Merge remote-tracking branch 'origin/main' into dbt-column-lineage-index

Round 7's findings, all in the frontend seam this branch added:

- A ducklake selection seeded the dbt fetch from the FIRST boundary relation it
  found, so a table derived from two unconnected dbt relations expanded one and
  left the other a leaf — the same "stops at the boundary" symptom the round-6
  fix removed, one hop further along. Every distinct boundary is fetched now and
  the components merged.
- The component cache skipped a relation merely PRESENT in the graph in hand.
  A relation two projects describe has an owner row in each, and a component
  fetched for one carries it as an endpoint without the other's half, so that
  skipped the request that would have resolved the second owner. Only a relation
  actually asked about under this pin is skipped.
- A comment still called the producer graph gated to ducklake selections after
  it was widened to dbt.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/.sqlx/query-06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n                               resource_type, name, asset_path, tags)\n         VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',\n                 'u/a/wh/analytics/stock', '{}'),\n                ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',\n                 'u/a/wh/analytics/stock_daily', '{}')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "06c1a79bfc24b17acbd79411295c718161d95f7ad65a26525c5f43241267608d"
+}

backend/.sqlx/query-1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n                                      parent_unique_id, parent_column, child_unique_id,\n                                      child_column, lineage_kind)\n         VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "1008ed150f30b56baf17b3c6e6bfc8ee144b14d564a3e589a60678b3e68effbf"
+}

backend/.sqlx/query-178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM dbt_column_edge\n          WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "178cb9cd8dfda66e878cd924f64a58ec742d7079703d417c886c8cf0a0e767a6"
+}

backend/.sqlx/query-210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n                                      parent_unique_id, parent_column, child_unique_id,\n                                      child_column, lineage_kind)\n         VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n                 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),\n                ($1, $2, $3, '00000000-0000-0000-0000-000000000000',\n                 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "210441eb7bee09afd27a927e0e44ff3bc1143dab285e51a9b54b2c2f240b9a8c"
+}

backend/.sqlx/query-2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n            resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n            tags, description, test_kind, test_column, test_args, severity, attached_node,\n            columns, column_schema, freshness, raw_code, original_file_path, ingested_at)\n         SELECT $2, script_path, script_hash, job_id, unique_id,\n            resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n            tags, description, test_kind, test_column, test_args, severity, attached_node,\n            columns, column_schema, freshness, raw_code, original_file_path, ingested_at\n         FROM dbt_node\n         WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "2e562fb35cfa71702e5fdbc505d2fa6f3114a151b66b0a116b1d98dd84115188"
+}

backend/.sqlx/query-3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n            parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,\n            ingested_at)\n         SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,\n            child_unique_id, child_column, lineage_kind, ingested_at\n         FROM dbt_column_edge\n         WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3b7858e47e4e3f31f861e114c7ff26b467185edcfde151f22c6e903d15722ad4"
+}

backend/.sqlx/query-7ae6da193977df13ec318c5b22b04101dedeb9a29dadf7dd8db1f69d02d93b45.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_edge (workspace_id, script_path, script_hash, job_id,\n                               parent_unique_id, child_unique_id)\n         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model.p.orders')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "7ae6da193977df13ec318c5b22b04101dedeb9a29dadf7dd8db1f69d02d93b45"
+}

backend/.sqlx/query-895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n                                      parent_unique_id, parent_column, child_unique_id,\n                                      child_column, lineage_kind)\n         VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "895a4feb0b3cc01ad711ce0eff8eef4f68cc3ecaf680c55a94175f6dfc2e2912"
+}

backend/.sqlx/query-94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n            resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n            tags, description, test_kind, test_column, test_args, severity, attached_node,\n            columns, freshness, raw_code, original_file_path, ingested_at)\n         SELECT $2, script_path, script_hash, job_id, unique_id,\n            resource_type, name, asset_path, materialized, materialize_strategy, unique_key,\n            tags, description, test_kind, test_column, test_args, severity, attached_node,\n            columns, freshness, raw_code, original_file_path, ingested_at\n         FROM dbt_node\n         WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Varchar"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "94c0aae349172b9295b81b9e61e2f8f0ca36920ab66e8922815157a0c147bc47"
-}

backend/.sqlx/query-97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "97fd4f9939d96176f4cfc4a07e69466279aad62f33c57f392835e8265ed40f26"
+}

backend/.sqlx/query-9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n                               resource_type, name, asset_path, tags)\n         VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',\n                 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),\n                ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',\n                 'model', 'orders', 'u/a/wh/analytics/orders', '{}')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9f0979110f86dffc7452d80ea0410ba39025d9c9787a71a9456241cf8a9b9d82"
+}

backend/.sqlx/query-b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n                               resource_type, name, asset_path, tags)\n         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',\n                 'u/a/wh/analytics/raw_orders', '{}')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b9c200365ea426ebe01b2e67c378ca41990e4c5d49b5baaee41acfe409bb1c24"
+}

backend/.sqlx/query-c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n                                 resource_type, name, asset_path, raw_code, tags)\n           VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n                   'u/a/wh/analytics/draft', 'select 3', '{}')",
+  "query": "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,\n                                 resource_type, name, asset_path, raw_code, tags)\n           VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',\n                   'u/a/wh/analytics/draft', 'select 3', '{}'),\n                  ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',\n                   'u/a/wh/analytics/draft_src', 'select 4', '{}')",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "1fb5590cb1fe706b0d2dffcd04a41bb2fce6012b812d16d37c895b2eb9aeca4c"
+  "hash": "c793b147014cb1b6e138aed1c2eb8b93b4a8383d8cf835a7d350056565669e7f"
 }

backend/.sqlx/query-e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM dbt_column_edge\n          WHERE job_id <> '00000000-0000-0000-0000-000000000000'\n            AND ingested_at < now() - make_interval(days => $1)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e5417b36319dafd5fcb2e42d812f778b2d3cf8b0f13cf4620192604cc3824a77"
+}

backend/.sqlx/query-f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2\n           AND script_hash = $3 AND job_id = $4",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f49cd9b5ea1e94d7cd10dcd1706cad62b5c1eff191f06905a3a78eb600e3bb4b"
+}

backend/.sqlx/query-f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,\n                                      parent_unique_id, parent_column, child_unique_id,\n                                      child_column, lineage_kind)\n         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',\n                 'copy')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Int8",
+        "Uuid"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f6de1512fa3e46883b32d56fd19ffe8aaf6dfe664c330ac8cea51a157a4fe52e"
+}

backend/Cargo.lock

@@ -16699,6 +16699,7 @@ dependencies = [
  "opentelemetry 0.30.0",
  "opentelemetry-proto 0.30.0",
  "oracle",
+ "parquet",
  "pem 3.0.6",
  "pep440_rs",
  "postgres-native-tls 0.5.3",

backend/Cargo.toml

@@ -665,6 +665,12 @@ process-wrap = { version = "8.2.1", features = ["tokio1"] }
 systemstat = "0.2.4"
 
 datafusion = "47.0.0"
+# The row API only: a dbt engine's parquet index is six string columns, so this
+# needs no arrow and no writer. `parquet` is already in the tree with `arrow` for
+# every shipped edition (`oss_core`), and cargo unifies the features there; this
+# set is what a build WITHOUT object storage compiles. ZSTD is what the engine
+# writes today, snap what parquet writers most often default to.
+parquet = { version = "55.2.0", default-features = false, features = ["snap", "zstd"] }
 object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] }
 openidconnect = { version = "4.0.0-rc.1" }
 aws-config = "^1"

backend/migrations/20260904143633_dbt_column_lineage.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS dbt_column_edge;
+ALTER TABLE dbt_node DROP COLUMN IF EXISTS column_schema;

backend/migrations/20260904143633_dbt_column_lineage.up.sql

@@ -0,0 +1,62 @@
+-- Column-level lineage, from the engine's own static analysis.
+--
+-- `manifest.json` carries none, which is why decision 14 recorded the feature as
+-- unavailable. The edges exist in a different artifact: an engine that does
+-- static analysis writes `target/index/dbt.column_lineage.parquet` under
+-- `dbt compile --static-analysis strict --write-index`. That pass is opt-in per
+-- project (`column_lineage: true`), because strict analysis rejects SQL the
+-- default accepts and must never become a silent requirement of running a build.
+
+-- One column-to-column edge, keyed exactly like `dbt_edge`: a version's graph
+-- dies with its version through the composite foreign key, a run's snapshot is
+-- keyed by `job_id` with the zero UUID meaning "the version's own graph", and an
+-- editor buffer's parse carries a NULL `script_hash` keyed to its preview job.
+CREATE TABLE IF NOT EXISTS dbt_column_edge (
+  workspace_id     VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  script_path      VARCHAR(255) NOT NULL,
+  script_hash      BIGINT,
+  job_id           UUID NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000',
+  parent_unique_id TEXT NOT NULL,
+  parent_column    TEXT NOT NULL,
+  child_unique_id  TEXT NOT NULL,
+  child_column     TEXT NOT NULL,
+  -- dbt's own word for how the value travelled: `copy` (passthrough), `mod`
+  -- (transformed), `scan` (the column was read to produce the ROW rather than
+  -- the value -- a join key, a `where` predicate, a `group by`). TEXT rather
+  -- than an enum because the engine treats the set as open: its own reader maps
+  -- those three and returns anything else verbatim.
+  lineage_kind     TEXT NOT NULL,
+  ingested_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Two partial unique indexes rather than a primary key, for the reason
+  -- 20260801121717 gives: a versioned graph is keyed by its version, a buffer
+  -- parse by its job alone.
+  CONSTRAINT dbt_column_edge_script_fkey FOREIGN KEY (workspace_id, script_hash)
+    REFERENCES script (workspace_id, hash) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_versioned_key
+  ON dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+                      parent_unique_id, parent_column, child_unique_id, child_column)
+  WHERE script_hash IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS dbt_column_edge_editor_key
+  ON dbt_column_edge (workspace_id, job_id,
+                      parent_unique_id, parent_column, child_unique_id, child_column)
+  WHERE script_hash IS NULL;
+
+-- Same age sweep as the other per-run rows, and the same reason there is no
+-- foreign key to `v2_job`.
+CREATE INDEX IF NOT EXISTS idx_dbt_column_edge_run_age ON dbt_column_edge (ingested_at)
+  WHERE job_id <> '00000000-0000-0000-0000-000000000000';
+
+-- The real column schema of a node, which only static analysis knows: an
+-- ordered `[{"name": …, "type": …}]`, from `dbt.node_columns.parquet`.
+--
+-- Beside `columns` rather than folded into it. `columns` is the DECLARED
+-- metadata `manifest.json` carries -- the names an author wrote in `schema.yml`
+-- and the prose against them -- and stays exactly that, so a project that
+-- documents two of forty columns keeps saying so. This is the other forty,
+-- typed, in the order the model produces them.
+ALTER TABLE dbt_node ADD COLUMN IF NOT EXISTS column_schema JSONB;
+
+GRANT ALL ON dbt_column_edge TO windmill_user;
+GRANT ALL ON dbt_column_edge TO windmill_admin;

backend/parsers/windmill-parser-yaml/src/dbt.rs

@@ -58,6 +58,19 @@ impl DbtEngine {
     pub fn emits_node_events(&self) -> bool {
         matches!(self, DbtEngine::DbtCore1x)
     }
+
+    /// Whether the engine's CLI has `--write-index`, the flag that writes the
+    /// parquet index column lineage lives in. False for 1.x, whose Python CLI
+    /// has no such option.
+    ///
+    /// True is not a promise that the artifact appears: `dbt-core` 2.0.0-alpha.5
+    /// accepts the flag, declares the views over `dbt.column_lineage` in its own
+    /// `views.sql`, and writes neither that parquet nor `dbt.node_columns`. Only
+    /// Fusion does today. Attempting the pass on both is what lets a later 2.x
+    /// release pick the feature up with no change here.
+    pub fn writes_column_index(&self) -> bool {
+        !matches!(self, DbtEngine::DbtCore1x)
+    }
 }
 
 /// How the warehouse connection is supplied. Both paths are supported
@@ -127,6 +140,18 @@ pub struct DbtDescriptor {
     pub selector: Option<String>,
     #[serde(default)]
     pub test_behavior: DbtTestBehavior,
+    /// Ingest column-to-column lineage and the real column schemas, from the
+    /// engine's static analysis.
+    ///
+    /// Opt-in, and it has to be: the artifact only appears under
+    /// `--static-analysis strict`, which rejects SQL the default accepts (an
+    /// unresolvable identifier is an error there and compiles fine otherwise).
+    /// Turning it on for everyone would make a stricter dialect the price of
+    /// deploying a dbt project. It is a separate `dbt compile` pass, so nothing
+    /// it decides can change what a build does; a project it cannot analyze
+    /// keeps the graph it has today.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub column_lineage: bool,
     /// `--vars`. dbt vars are typed — numbers, booleans, lists and objects are
     /// all normal — so values keep their YAML type; only string leaves carry
     /// `{{ arg }}` placeholders the worker substitutes from job args. Coercing

backend/summarized_schema.txt

@@ -70,11 +70,13 @@ ci_test_reference: workspace_id(char), test_script_path(char), test_script_hash(
 concurrency_settings: hash(bigint), concurrency_key(char), concurrent_limit(int), concurrency_time_window_s(int)
 config: name(char), config(jsonb)
 custom_concurrency_key_ended: key(char), ended_at(ts)
+dbt_column_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), parent_column(text), child_unique_id(text), child_column(text), lineage_kind(text), ingested_at(ts)
+  FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash)
 dbt_edge: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), parent_unique_id(text), child_unique_id(text), ingested_at(ts)
   FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash)
 dbt_graph_snapshot: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), digest(text), relation_root_at_last_ingest(text), ingested_at(ts), permissioned_as(char)
   FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash)
-dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts)
+dbt_node: workspace_id(char), script_path(char), script_hash(bigint), job_id(uuid), unique_id(text), resource_type(text), name(text), asset_path(text), materialized(text), materialize_strategy(text), unique_key(text), tags(text[]), description(text), test_kind(text), test_column(text), test_args(jsonb), severity(text), attached_node(text), columns(jsonb), column_schema(jsonb), freshness(jsonb), raw_code(text), original_file_path(text), ingested_at(ts)
   FK: (workspace_id) -> workspace(id), (workspace_id, script_hash) -> script(workspace_id, hash)
 dbt_run_progress: workspace_id(char), job_id(uuid), asset_kind(asset_kind), asset_path(char), status(materialization_status), row_count(bigint), error(text), updated_at(ts)
   FK: (workspace_id) -> workspace(id)

backend/windmill-api-assets/src/lib.rs

@@ -6,6 +6,7 @@ use axum::{
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use sqlx::Row;
+use std::collections::{HashMap, HashSet};
 use windmill_common::{
     assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind},
     db::UserDB,
@@ -13,7 +14,9 @@ use windmill_common::{
     utils::escape_ilike_pattern,
 };
 
-use windmill_api_auth::{build_scope_path_predicate, ApiAuthed};
+use windmill_api_auth::{
+    build_scope_path_filter, build_scope_path_predicate, ApiAuthed, ScopePathFilter,
+};
 
 // Partition-range backfill preview. The logic (producer resolution, range
 // enumeration, status join) is enterprise: the `private` build compiles the
@@ -33,6 +36,7 @@ pub fn workspaced_service() -> Router {
         .route("/list_by_usages", post(list_assets_by_usages))
         .route("/list_favorites", get(list_favorites))
         .route("/graph", get(asset_graph))
+        .route("/column_lineage", get(dbt_column_lineage))
         .route("/pipelines", get(list_pipeline_folders))
         .route("/partitions", get(list_partitions))
         .route("/partitions_in_range", get(list_partitions_in_range))
@@ -663,10 +667,21 @@ struct DbtAssetProvenance {
     description: Option<String>,
     #[serde(skip_serializing_if = "Vec::is_empty")]
     data_tests: Vec<DbtDataTest>,
-    /// Declared column metadata (name -> description). NOT column lineage —
-    /// `manifest.json` carries none (docs/dbt-runtime.md, decision 14).
+    /// Declared column metadata (name -> description): what `manifest.json`
+    /// carries, which is only the columns an author wrote down.
     #[serde(skip_serializing_if = "Option::is_none")]
     columns: Option<serde_json::Value>,
+    /// Every column of the relation, typed and in order —
+    /// `[{"name": …, "type": …}]` — from the engine's static analysis. Present
+    /// only for a project that opted into it.
+    ///
+    /// Gated exactly like `columns` and the model's SQL: a full column list is
+    /// the shape of what the author WROTE, one level finer than the `ref()`
+    /// graph, which is ungated only because it draws relations the caller
+    /// already sees in `asset`. Widening that boundary has to be a decision, not
+    /// a consequence of a project turning the analysis pass on.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    column_schema: Option<serde_json::Value>,
     /// A source's declared freshness policy, for the staleness chip.
     #[serde(skip_serializing_if = "Option::is_none")]
     freshness: Option<serde_json::Value>,
@@ -946,6 +961,266 @@ struct DbtLineageEdge {
     to_asset_path: String,
 }
 
+/// One column-to-column edge, in the same terms: the two relations and the two
+/// columns, never dbt's node ids.
+#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct DbtColumnLineageEdge {
+    from_asset_path: String,
+    from_column: String,
+    to_asset_path: String,
+    to_column: String,
+    /// dbt's own word for how the value travelled: `copy` (passthrough), `mod`
+    /// (transformed), `scan` (read to produce the ROW rather than the value — a
+    /// join key, a predicate, a `group by`). Sent verbatim, including a kind
+    /// this engine version invented, because the renderer decides what a kind
+    /// means and the set is the engine's.
+    kind: String,
+}
+
+/// One dbt relation's column lineage: the connected component its columns sit
+/// in, within the project that owns it.
+///
+/// Not the relation's own edges, which would stop one hop out — a trace walks
+/// transitively — and not the whole project's, which carries families the
+/// selected relation cannot reach. The component is what the canvas lays out,
+/// so it is exactly what a consumer can draw.
+///
+/// Its own endpoint rather than a field on the graph, which is folder-wide and
+/// polled by a run page while this is rendered for a single selection. A
+/// folder's worth of edges spans many projects and many callers' access, so it
+/// would need a cap, and a cap has to be applied after every filter that could
+/// drop a row — which is the ordering this shape removes rather than gets
+/// right. Here the filters ARE the answer: scope and visibility are decided
+/// once in SQL for the script that owns the relation, the component is walked
+/// over what that returns, and the size is bounded at ingest
+/// (`MAX_COLUMN_EDGES` per version, of which only the direct kinds are served).
+#[derive(Deserialize)]
+pub struct ColumnLineageQuery {
+    /// The `dbt://` relation whose lineage to return.
+    pub asset_path: String,
+    /// A deployed version other than the current one, for an editor open on an
+    /// older one. A run's or an editor buffer's graph is NOT reachable from
+    /// here: it pins to a job, and that costs the job-read gate.
+    pub dbt_script_hash: Option<windmill_common::scripts::ScriptHash>,
+}
+
+#[derive(Serialize, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ColumnLineageResponse {
+    /// Direct (`copy` / `mod`) column edges of the component this relation's
+    /// columns sit in, in the terms the canvas draws. Empty when the project
+    /// never asked for the analysis pass, which is the ordinary case.
+    edges: Vec<DbtColumnLineageEdge>,
+}
+
+async fn dbt_column_lineage(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+    Query(q): Query<ColumnLineageQuery>,
+) -> JsonResult<ColumnLineageResponse> {
+    // `None`: pinning to one run is job-scoped and this endpoint is authorized
+    // as `assets:read`. See `dbt_column_lineage_for`.
+    dbt_column_lineage_for(&authed, &w_id, user_db, q, None).await
+}
+
+/// One dbt relation's column lineage, optionally as one run saw it.
+///
+/// AUTHORIZES NOTHING BY ITSELF, on the same contract as `asset_graph_for`:
+/// `assets:read` always, and the job-read gate for `Some(pinned)`, whose path
+/// and hash are then taken from that job's row rather than from the caller.
+pub async fn dbt_column_lineage_for(
+    authed: &ApiAuthed,
+    w_id: &str,
+    user_db: UserDB,
+    q: ColumnLineageQuery,
+    pinned: Option<PinnedRun>,
+) -> JsonResult<ColumnLineageResponse> {
+    // A column-level view is the shape of what the author WROTE, so it takes the
+    // model's own gate rather than the relation's. Decided ONCE here, for the
+    // script that owns the asset, instead of per row against a cap.
+    let (scope_all, scope_exact, scope_prefix) =
+        match build_scope_path_filter(authed, "scripts", "read") {
+            ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()),
+            ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix),
+        };
+    let (pinned_path, script_hash) = match pinned.as_ref() {
+        // The job's own version, so a pin cannot name one project's run while
+        // claiming another's version — including when it names NONE, which is
+        // the editor buffer.
+        Some(p) => (Some(p.script_path.as_str()), p.script_hash),
+        None => (None, q.dbt_script_hash.map(|h| h.0)),
+    };
+    let pinned_job_id = pinned.as_ref().map(|p| p.job_id);
+    let mut tx = user_db.begin(authed).await?;
+    let rows = sqlx::query!(
+        r#"WITH
+           -- The project version that owns the asked-for relation, in the graph
+           -- on screen. Not the folder-wide `live` set the graph resolves: one
+           -- asset is asked about here, so the version is decided per candidate
+           -- row. Usually one row; a relation a second project declares as a
+           -- source has two, and each answers for its own lineage.
+           owner AS (
+             SELECT DISTINCT n.script_path, n.script_hash, n.job_id
+               FROM dbt_node n
+              WHERE n.workspace_id = $1 AND n.asset_path = $2
+                -- The run's snapshot, or the deployed graph when that job stored
+                -- none -- a build pins only if it wrote one.
+                AND n.job_id = CASE WHEN $5::uuid IS NOT NULL AND EXISTS (
+                                      SELECT 1 FROM dbt_graph_snapshot g
+                                       WHERE g.workspace_id = $1 AND g.job_id = $5)
+                                    THEN $5::uuid
+                                    ELSE '00000000-0000-0000-0000-000000000000'::uuid END
+                -- The gate, once, for the whole answer.
+                AND ( $6
+                      OR n.script_path = ANY($7)
+                      OR EXISTS ( SELECT 1 FROM unnest($8::text[]) AS pfx
+                                   WHERE n.script_path = pfx
+                                      OR left(n.script_path, length(pfx) + 1) = pfx || '/' ) )
+                AND CASE
+                      -- Pinned: which version comes from a job this caller was
+                      -- already granted, so `script` does not decide THAT — but
+                      -- it still decides whether the project may be read, the
+                      -- same second gate `script_visible` is on the graph. Being
+                      -- entitled to a run is not being entitled to the SQL
+                      -- behind it, and column lineage is that SQL's shape. A
+                      -- version-less row is exempt because it is an editor
+                      -- buffer, which has no `script` row to ask and reaches
+                      -- this only through the parse job that wrote it.
+                      WHEN $4::text IS NOT NULL
+                        THEN n.script_path = $4 AND n.script_hash IS NOT DISTINCT FROM $3::bigint
+                             AND ($3::bigint IS NULL OR EXISTS (
+                                   SELECT 1 FROM script sc
+                                    WHERE sc.workspace_id = $1 AND sc.path = n.script_path
+                                      AND sc.hash = $3))
+                      -- A named version, for an editor open on an older one.
+                      -- `script` is read under RLS, so this is the visibility
+                      -- check as well as the existence one.
+                      WHEN $3::bigint IS NOT NULL
+                        THEN n.script_hash = $3 AND EXISTS (
+                               SELECT 1 FROM script sc
+                                WHERE sc.workspace_id = $1 AND sc.path = n.script_path
+                                  AND sc.hash = $3)
+                      -- Otherwise the version deployed now: an older one's rows
+                      -- outlive it in `dbt_node` until the sweep, and describe a
+                      -- project that is no longer what runs. `language` narrows
+                      -- it the way the graph's own resolution does, so a path
+                      -- that has since become a script of another kind draws and
+                      -- explains the same version rather than disagreeing.
+                      ELSE n.script_hash = (
+                             SELECT sc.hash FROM script sc
+                              WHERE sc.workspace_id = $1 AND sc.path = n.script_path
+                                AND sc.language = 'dbt'
+                                AND sc.deleted = false AND sc.archived = false
+                              ORDER BY sc.created_at DESC LIMIT 1)
+                    END
+           )
+           -- DIRECT kinds only. `scan` — the column was read to produce the ROW,
+           -- not the value — reaches every output column of its model, so it is
+           -- most of a project's stored lineage and none of what a trace draws.
+           -- It stays in the table for a later view to ask for.
+           SELECT p.asset_path AS "from_path!", e.parent_column AS "from_column!",
+                  c.asset_path AS "to_path!", e.child_column AS "to_column!",
+                  e.lineage_kind AS "kind!"
+             FROM dbt_column_edge e
+             JOIN owner o ON o.script_path = e.script_path
+                         AND o.script_hash IS NOT DISTINCT FROM e.script_hash
+                         AND o.job_id = e.job_id
+             JOIN dbt_node p ON p.workspace_id = e.workspace_id
+                            AND p.script_path = e.script_path
+                            AND p.script_hash IS NOT DISTINCT FROM e.script_hash
+                            AND p.job_id = e.job_id
+                            AND p.unique_id = e.parent_unique_id
+             JOIN dbt_node c ON c.workspace_id = e.workspace_id
+                            AND c.script_path = e.script_path
+                            AND c.script_hash IS NOT DISTINCT FROM e.script_hash
+                            AND c.job_id = e.job_id
+                            AND c.unique_id = e.child_unique_id
+            WHERE e.workspace_id = $1
+              AND e.lineage_kind IN ('copy', 'mod')
+              AND p.asset_path IS NOT NULL AND c.asset_path IS NOT NULL"#,
+        w_id,
+        q.asset_path,
+        script_hash,
+        pinned_path,
+        pinned_job_id,
+        scope_all,
+        &scope_exact[..],
+        &scope_prefix[..],
+    )
+    .fetch_all(&mut *tx)
+    .await?;
+    tx.commit().await?;
+
+    let mut edges: Vec<DbtColumnLineageEdge> = rows
+        .into_iter()
+        .map(|r| DbtColumnLineageEdge {
+            from_asset_path: r.from_path,
+            from_column: r.from_column,
+            to_asset_path: r.to_path,
+            to_column: r.to_column,
+            kind: r.kind,
+        })
+        .collect();
+    // Two projects can describe one relation, so the same edge can arrive twice.
+    edges.sort();
+    edges.dedup();
+    Ok(Json(ColumnLineageResponse {
+        edges: component(edges, &q.asset_path),
+    }))
+}
+
+/// Keep the edges of the connected component the asked-for relation sits in.
+///
+/// The canvas lays out the component of the selected relation's columns, so a
+/// project's other model families are edges nothing it draws can reach. Walked
+/// here rather than in SQL: a recursive CTE has no index to walk, so it rescans
+/// the whole edge set once per level — measured at 1.24s against 59ms for the
+/// query alone on a 3000-model project, for a walk that is microseconds over a
+/// map. Columns are keyed by relation, not by project, which is how the canvas
+/// keys them too: two projects describing one relation draw one node.
+fn component(mut edges: Vec<DbtColumnLineageEdge>, asset_path: &str) -> Vec<DbtColumnLineageEdge> {
+    let keep = {
+        let mut incident: HashMap<(&str, &str), Vec<usize>> = HashMap::new();
+        for (i, e) in edges.iter().enumerate() {
+            let from = (e.from_asset_path.as_str(), e.from_column.as_str());
+            let to = (e.to_asset_path.as_str(), e.to_column.as_str());
+            incident.entry(from).or_default().push(i);
+            incident.entry(to).or_default().push(i);
+        }
+        let mut stack: Vec<(&str, &str)> = incident
+            .keys()
+            .filter(|(path, _)| *path == asset_path)
+            .copied()
+            .collect();
+        let mut seen_node: HashSet<(&str, &str)> = stack.iter().copied().collect();
+        let mut seen_edge = vec![false; edges.len()];
+        while let Some(node) = stack.pop() {
+            for &i in incident.get(&node).map(Vec::as_slice).unwrap_or_default() {
+                if std::mem::replace(&mut seen_edge[i], true) {
+                    continue;
+                }
+                let e = &edges[i];
+                let ends = [
+                    (e.from_asset_path.as_str(), e.from_column.as_str()),
+                    (e.to_asset_path.as_str(), e.to_column.as_str()),
+                ];
+                for end in ends {
+                    if seen_node.insert(end) {
+                        stack.push(end);
+                    }
+                }
+            }
+        }
+        seen_edge
+    };
+    let mut i = 0;
+    edges.retain(|_| {
+        i += 1;
+        keep[i - 1]
+    });
+    edges
+}
+
 async fn asset_graph(
     authed: ApiAuthed,
     Path(w_id): Path<String>,
@@ -1323,7 +1598,7 @@ pub async fn asset_graph_for(
                   n.resource_type AS "resource_type!", n.name AS "name!", n.asset_path,
                   n.materialized, n.materialize_strategy, n.tags AS "tags!", n.description,
                   n.test_kind, n.test_column, n.test_args, n.severity, n.attached_node,
-                  n.columns, n.freshness,
+                  n.columns, n.column_schema, n.freshness,
                   n.raw_code, n.original_file_path,
                   -- Whether the caller may read the project this row describes.
                   -- The query deliberately reaches outside the requested folder
@@ -1379,6 +1654,10 @@ pub async fn asset_graph_for(
     // `ref()` lineage between two models, resolved to the relations they
     // produce. Joined to `dbt_node` on both key columns because a dbt
     // `unique_id` is only unique within its project.
+    //
+    // Column lineage is NOT here: it is per-selected-asset and served by
+    // `/column_lineage`, so this response — which a run page polls — carries
+    // only what the canvas draws for every node at once.
     let dbt_edge_rows = sqlx::query!(
         r#"WITH live AS (
              SELECT * FROM (
@@ -1604,6 +1883,7 @@ pub async fn asset_graph_for(
             description: r.description.clone().filter(|_| source_allowed),
             data_tests: vec![],
             columns: r.columns.clone().filter(|_| source_allowed),
+            column_schema: r.column_schema.clone().filter(|_| source_allowed),
             freshness: r.freshness.clone().filter(|_| source_allowed),
         };
         // One relation can carry rows from several projects — typically a model

backend/windmill-api-assets/tests/dbt_pinned_graph.rs

@@ -8,7 +8,9 @@
 //! whole dbt half, and every later fix in this area re-touched one of the two.
 
 use sqlx::{Pool, Postgres};
-use windmill_api_assets::{asset_graph_for, GraphQuery, PinnedRun};
+use windmill_api_assets::{
+    asset_graph_for, dbt_column_lineage_for, ColumnLineageQuery, GraphQuery, PinnedRun,
+};
 use windmill_api_auth::ApiAuthed;
 use windmill_common::db::UserDB;
 
@@ -84,6 +86,47 @@ async fn seed(db: &Pool<Postgres>, job: uuid::Uuid) {
     .execute(db)
     .await
     .unwrap();
+    // The relation it reads, and the `ref()` between them — what draws the
+    // project as a DAG rather than a fan-out off the one dbt runnable.
+    sqlx::query!(
+        "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,
+                               resource_type, name, asset_path, tags)
+         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model', 'raw_orders',
+                 'u/a/wh/analytics/raw_orders', '{}')",
+        WS,
+        PATH,
+        HASH,
+        job
+    )
+    .execute(db)
+    .await
+    .unwrap();
+    sqlx::query!(
+        "INSERT INTO dbt_edge (workspace_id, script_path, script_hash, job_id,
+                               parent_unique_id, child_unique_id)
+         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'model.p.orders')",
+        WS,
+        PATH,
+        HASH,
+        job
+    )
+    .execute(db)
+    .await
+    .unwrap();
+    sqlx::query!(
+        "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+                                      parent_unique_id, parent_column, child_unique_id,
+                                      child_column, lineage_kind)
+         VALUES ($1, $2, $3, $4, 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id',
+                 'copy')",
+        WS,
+        PATH,
+        HASH,
+        job
+    )
+    .execute(db)
+    .await
+    .unwrap();
     // A test node, for the arguments it carries: `accepted_values` spells out a
     // column's domain.
     sqlx::query!(
@@ -162,6 +205,14 @@ async fn a_pinned_run_survives_no_access_to_its_script(db: Pool<Postgres>) {
         body.to_string().contains("u/a/wh/analytics/orders"),
         "while the relation the run wrote is what the page is for: {body}"
     );
+    assert_eq!(
+        body["dbt_edges"],
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/raw_orders",
+            "to_asset_path": "u/a/wh/analytics/orders",
+        }]),
+        "and the `ref()` between them, resolved to relations: {body}"
+    );
 
     // The same read by someone who may open the project: the gate has to be the
     // caller's access, not a field this endpoint stopped serving.
@@ -365,7 +416,25 @@ async fn seed_editor_graph(db: &Pool<Postgres>, job: uuid::Uuid) {
         r#"INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,
                                  resource_type, name, asset_path, raw_code, tags)
            VALUES ($1, $2, NULL, $3, 'model.p.draft', 'model', 'draft',
-                   'u/a/wh/analytics/draft', 'select 3', '{}')"#,
+                   'u/a/wh/analytics/draft', 'select 3', '{}'),
+                  ($1, $2, NULL, $3, 'model.p.draft_src', 'model', 'draft_src',
+                   'u/a/wh/analytics/draft_src', 'select 4', '{}')"#,
+        WS,
+        PATH,
+        job
+    )
+    .execute(db)
+    .await
+    .unwrap();
+    // A version-less row's `script_hash` is NULL on both sides of every join and
+    // every visibility check, and `= NULL` is never true — so the column edges
+    // need the same NULL arm the node query has, or a buffer parse renders its
+    // columns and none of their lineage.
+    sqlx::query!(
+        "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+                                      parent_unique_id, parent_column, child_unique_id,
+                                      child_column, lineage_kind)
+         VALUES ($1, $2, NULL, $3, 'model.p.draft_src', 'raw', 'model.p.draft', 'clean', 'mod')",
         WS,
         PATH,
         job
@@ -451,3 +520,216 @@ async fn an_editor_graph_renders_only_through_its_own_job(db: Pool<Postgres>) {
         "nor of a run of the deployed version: {deployed_run}"
     );
 }
+
+fn column_query(asset_path: &str) -> ColumnLineageQuery {
+    ColumnLineageQuery { asset_path: asset_path.to_string(), dbt_script_hash: None }
+}
+
+async fn column_lineage(
+    db: &Pool<Postgres>,
+    authed: &ApiAuthed,
+    asset_path: &str,
+    pinned: Option<PinnedRun>,
+) -> serde_json::Value {
+    let res = dbt_column_lineage_for(
+        authed,
+        WS,
+        UserDB::new(db.clone()),
+        column_query(asset_path),
+        pinned,
+    )
+    .await
+    .unwrap();
+    serde_json::to_value(&res.0).unwrap()["edges"].clone()
+}
+
+/// The buffer parse's own lineage, which is the case the versionless rows exist
+/// for. Its `script_hash` is NULL on both sides of every join and every
+/// visibility check, and `= NULL` is never true — so the versionless arm has to
+/// be written for it, or a parse renders its columns and none of their lineage.
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn an_editor_buffers_column_lineage_answers_through_its_job(db: Pool<Postgres>) {
+    let parse = uuid::Uuid::from_u128(9);
+    seed(&db, uuid::Uuid::from_u128(7)).await;
+    seed_editor_graph(&db, parse).await;
+    let admin = ApiAuthed { is_admin: true, ..outsider() };
+
+    let pinned = PinnedRun { job_id: parse, script_path: PATH.to_string(), script_hash: None };
+    assert_eq!(
+        column_lineage(&db, &admin, "u/a/wh/analytics/draft", Some(pinned)).await,
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/draft_src",
+            "from_column": "raw",
+            "to_asset_path": "u/a/wh/analytics/draft",
+            "to_column": "clean",
+            "kind": "mod",
+        }]),
+    );
+    // Unpinned, the same relation resolves through the deployed version, which
+    // never heard of the buffer's models.
+    assert_eq!(
+        column_lineage(&db, &admin, "u/a/wh/analytics/draft", None).await,
+        serde_json::json!([]),
+        "a buffer's lineage is reachable only through the job that parsed it"
+    );
+}
+
+/// Being entitled to a RUN is not being entitled to the SQL behind it, and
+/// column lineage is that SQL's shape. The pinned graph draws the relations for
+/// a share-link viewer and redacts what the author wrote; the lineage is the
+/// second, and resolving the version from the job must not be mistaken for
+/// deciding that too.
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn a_pinned_run_does_not_hand_over_the_projects_column_lineage(db: Pool<Postgres>) {
+    let job = uuid::Uuid::from_u128(7);
+    seed(&db, job).await;
+    let pinned =
+        || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) };
+
+    assert_eq!(
+        column_lineage(&db, &outsider(), "u/a/wh/analytics/orders", Some(pinned())).await,
+        serde_json::json!([]),
+        "the run renders for them, its column-level shape does not"
+    );
+    assert_eq!(
+        column_lineage(
+            &db,
+            &ApiAuthed { is_admin: true, ..outsider() },
+            "u/a/wh/analytics/orders",
+            Some(pinned())
+        )
+        .await,
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/raw_orders",
+            "from_column": "id",
+            "to_asset_path": "u/a/wh/analytics/orders",
+            "to_column": "order_id",
+            "kind": "copy",
+        }]),
+        "while a reader of the project gets it"
+    );
+}
+
+/// A column-level view is the shape of what the author WROTE, so it takes the
+/// script's own gate — the same one that keeps `raw_code` behind access to the
+/// project, applied here once for the script that owns the relation.
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn column_lineage_takes_the_scripts_gate_and_only_the_direct_kinds(db: Pool<Postgres>) {
+    let job = uuid::Uuid::from_u128(7);
+    seed(&db, job).await;
+    // A second relation of the deployed version, feeding the first. `scan` says
+    // the column was read to produce the ROW rather than the value, so it
+    // reaches every output column of its model and is never served.
+    sqlx::query!(
+        "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,
+                               resource_type, name, asset_path, tags)
+         VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.raw_orders',
+                 'model', 'raw_orders', 'u/a/wh/analytics/raw_orders', '{}'),
+                ($1, $2, $3, '00000000-0000-0000-0000-000000000000', 'model.p.orders',
+                 'model', 'orders', 'u/a/wh/analytics/orders', '{}')",
+        WS,
+        PATH,
+        HASH,
+    )
+    .execute(&db)
+    .await
+    .unwrap();
+    sqlx::query!(
+        "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+                                      parent_unique_id, parent_column, child_unique_id,
+                                      child_column, lineage_kind)
+         VALUES ($1, $2, $3, '00000000-0000-0000-0000-000000000000',
+                 'model.p.raw_orders', 'id', 'model.p.orders', 'order_id', 'copy'),
+                ($1, $2, $3, '00000000-0000-0000-0000-000000000000',
+                 'model.p.raw_orders', 'status', 'model.p.orders', 'order_id', 'scan')",
+        WS,
+        PATH,
+        HASH,
+    )
+    .execute(&db)
+    .await
+    .unwrap();
+
+    let admin = ApiAuthed { is_admin: true, ..outsider() };
+    assert_eq!(
+        column_lineage(&db, &admin, "u/a/wh/analytics/orders", None).await,
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/raw_orders",
+            "from_column": "id",
+            "to_asset_path": "u/a/wh/analytics/orders",
+            "to_column": "order_id",
+            "kind": "copy",
+        }]),
+        "the direct edge, and not the `scan` one beside it"
+    );
+    assert_eq!(
+        column_lineage(&db, &outsider(), "u/a/wh/analytics/orders", None).await,
+        serde_json::json!([]),
+        "and nothing at all for a caller who cannot read the project"
+    );
+}
+
+/// One project routinely holds model families that share no column, and the
+/// canvas lays out the connected component of the selected relation's columns.
+/// Answering with the project's other components sends edges nothing can draw.
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn column_lineage_stops_at_the_selected_relations_component(db: Pool<Postgres>) {
+    let job = uuid::Uuid::from_u128(7);
+    seed(&db, job).await;
+    // A second family in the same project version, reaching neither of the two
+    // relations `seed` wired together.
+    sqlx::query!(
+        "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,
+                               resource_type, name, asset_path, tags)
+         VALUES ($1, $2, $3, $4, 'model.p.stock', 'model', 'stock',
+                 'u/a/wh/analytics/stock', '{}'),
+                ($1, $2, $3, $4, 'model.p.stock_daily', 'model', 'stock_daily',
+                 'u/a/wh/analytics/stock_daily', '{}')",
+        WS,
+        PATH,
+        HASH,
+        job
+    )
+    .execute(&db)
+    .await
+    .unwrap();
+    sqlx::query!(
+        "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+                                      parent_unique_id, parent_column, child_unique_id,
+                                      child_column, lineage_kind)
+         VALUES ($1, $2, $3, $4, 'model.p.stock', 'sku', 'model.p.stock_daily', 'sku', 'copy')",
+        WS,
+        PATH,
+        HASH,
+        job
+    )
+    .execute(&db)
+    .await
+    .unwrap();
+
+    let admin = ApiAuthed { is_admin: true, ..outsider() };
+    let pinned =
+        || PinnedRun { job_id: job, script_path: PATH.to_string(), script_hash: Some(HASH) };
+    assert_eq!(
+        column_lineage(&db, &admin, "u/a/wh/analytics/orders", Some(pinned())).await,
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/raw_orders",
+            "from_column": "id",
+            "to_asset_path": "u/a/wh/analytics/orders",
+            "to_column": "order_id",
+            "kind": "copy",
+        }]),
+        "the orders family, and not the stock one beside it in the same project"
+    );
+    assert_eq!(
+        column_lineage(&db, &admin, "u/a/wh/analytics/stock_daily", Some(pinned())).await,
+        serde_json::json!([{
+            "from_asset_path": "u/a/wh/analytics/stock",
+            "from_column": "sku",
+            "to_asset_path": "u/a/wh/analytics/stock_daily",
+            "to_column": "sku",
+            "kind": "copy",
+        }]),
+        "and the other way round — reached from the child end, which is upstream"
+    );
+}

backend/windmill-api-workspaces/src/workspaces.rs

@@ -6627,7 +6627,7 @@ async fn clone_scripts(
 }
 
 /// The parsed dbt graph a deployed script carries: its models, their SQL and
-/// tests, and the `ref()` lineage between them.
+/// tests, and the `ref()` and column-level lineage between them.
 ///
 /// Keyed on (workspace_id, script_path, script_hash), and the fork keeps every
 /// script's hash, so each row moves across as itself.
@@ -6645,11 +6645,11 @@ async fn clone_dbt_graph(
         "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id,
             resource_type, name, asset_path, materialized, materialize_strategy, unique_key,
             tags, description, test_kind, test_column, test_args, severity, attached_node,
-            columns, freshness, raw_code, original_file_path, ingested_at)
+            columns, column_schema, freshness, raw_code, original_file_path, ingested_at)
          SELECT $2, script_path, script_hash, job_id, unique_id,
             resource_type, name, asset_path, materialized, materialize_strategy, unique_key,
             tags, description, test_kind, test_column, test_args, severity, attached_node,
-            columns, freshness, raw_code, original_file_path, ingested_at
+            columns, column_schema, freshness, raw_code, original_file_path, ingested_at
          FROM dbt_node
          WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'",
         source_workspace_id,
@@ -6669,6 +6669,24 @@ async fn clone_dbt_graph(
     )
     .execute(&mut **tx)
     .await?;
+    // Column lineage travels with the rest of the graph, and it has to: the
+    // snapshot's digest covers it, so a fork missing these rows recomputes the
+    // digest the source stored, matches, and stores nothing — leaving the
+    // lineage gone until someone redeploys, which is the failure this whole
+    // function exists to prevent.
+    sqlx::query!(
+        "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id,
+            parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind,
+            ingested_at)
+         SELECT $2, script_path, script_hash, job_id, parent_unique_id, parent_column,
+            child_unique_id, child_column, lineage_kind, ingested_at
+         FROM dbt_column_edge
+         WHERE workspace_id = $1 AND job_id = '00000000-0000-0000-0000-000000000000'",
+        source_workspace_id,
+        target_workspace_id
+    )
+    .execute(&mut **tx)
+    .await?;
     sqlx::query!(
         "INSERT INTO dbt_graph_snapshot (workspace_id, script_path, script_hash, job_id,
             digest, relation_root_at_last_ingest, ingested_at)

backend/windmill-api/openapi.yaml

@@ -24364,6 +24364,57 @@ paths:
               schema:
                 $ref: "#/components/schemas/AssetGraph"
 
+  /w/{workspace}/assets/column_lineage:
+    get:
+      summary: Column-level lineage of one dbt relation
+      description: >
+        The direct (`copy` / `mod`) column-to-column lineage a relation's columns
+        sit in — the connected component within the dbt project that owns it,
+        from the engine's static analysis. Not the relation's own edges, which
+        would stop one hop out since a column trace walks transitively, and not
+        the whole project's, which carries model families the selected relation
+        cannot reach. The component is what the canvas lays out.
+
+        Its own endpoint rather than a field on the asset graph: the graph is
+        folder-wide and polled by a run page, while this is rendered for one
+        selected asset at a time. A folder's worth of edges spans many projects
+        and many callers' access, so it would need a cap, and a cap has to come
+        after every filter that can drop a row. Here the filters are the answer:
+        the caller's scope and the project's visibility are decided once for the
+        owning script, and the size is bounded where the index is ingested.
+
+        Empty for a project that did not opt into the analysis pass
+        (`column_lineage: true`), which is the ordinary case. The indirect `scan`
+        kind is stored but never served: it reaches every output column of its
+        model.
+      operationId: getDbtColumnLineage
+      tags:
+        - asset
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: asset_path
+          in: query
+          required: true
+          description: The `dbt://` relation whose lineage to return
+          schema:
+            type: string
+        - name: dbt_script_hash
+          in: query
+          description: >
+            Pin to one deployed version, for an editor open on an older one. A
+            run's or an editor buffer's own graph is not reachable here: that
+            pins to a job, and costs the job-read gate — see
+            `jobs/dbt_column_lineage/{id}`.
+          schema:
+            type: string
+      responses:
+        "200":
+          description: the relation's column-level lineage
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DbtColumnLineage"
+
   /w/{workspace}/assets/macros:
     get:
       summary: List every workspace DuckDB macro (deployed `// macros` libraries)
@@ -24555,6 +24606,43 @@ paths:
               schema:
                 $ref: "#/components/schemas/AssetGraph"
 
+  /w/{workspace}/jobs/dbt_column_lineage/{id}:
+    get:
+      summary: Get a relation's project column lineage as one run saw it
+      description: >
+        The same answer as `assets/column_lineage`, for the project version a
+        single job ran — including the dbt editor's parse of its own buffer,
+        whose graph belongs to that job and is reachable no other way.
+        Authorized through the job, the same gate as `dbt_graph`. Reaching the
+        run is not on its own enough to read the project: a caller with no access
+        to the script gets its relations and `ref()` edges from `dbt_graph` and
+        an empty answer here, exactly as that endpoint redacts the model's SQL.
+      operationId: getDbtRunColumnLineage
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: id
+          in: path
+          required: true
+          description: The job whose graph the lineage is read from
+          schema:
+            type: string
+            format: uuid
+        - name: asset_path
+          in: query
+          required: true
+          description: The `dbt://` relation whose lineage to return
+          schema:
+            type: string
+      responses:
+        "200":
+          description: the relation's column-level lineage
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DbtColumnLineage"
+
   /w/{workspace}/jobs/run_progress/{id}:
     get:
       summary: List the per-relation progress one job has recorded so far
@@ -26148,6 +26236,34 @@ components:
             drawn identically and the ambiguity would otherwise just move into
             the editor. Omitted for the unpinned workspace graph, which spans
             every project and so has no one time.
+    DbtColumnLineage:
+      type: object
+      description: >-
+        The direct column-to-column lineage a relation's columns sit in — the
+        connected component within the project that owns it — in the terms the
+        canvas draws: relations and columns, never dbt's node ids.
+      required: [edges]
+      properties:
+        edges:
+          type: array
+          items:
+            type: object
+            required: [from_asset_path, from_column, to_asset_path, to_column, kind]
+            properties:
+              from_asset_path:
+                type: string
+              from_column:
+                type: string
+              to_asset_path:
+                type: string
+              to_column:
+                type: string
+              kind:
+                type: string
+                description: >-
+                  dbt's own word for how the value travelled — `copy`
+                  (passthrough) or `mod` (transformed). Not an enum: the engine
+                  treats the set as open.
     DbtAssetProvenance:
       type: object
       description: >-
@@ -26195,7 +26311,23 @@ components:
         columns:
           type: object
           additionalProperties: true
-          description: Declared column metadata (name -> description). NOT column lineage — `manifest.json` carries none.
+          description: Declared column metadata (name -> description) — what `manifest.json` carries, which is only the columns an author wrote down. Omitted when the caller cannot read the script.
+        column_schema:
+          type: array
+          description: >-
+            Every column of the relation, typed and in the order the model
+            produces them, from the engine's static analysis. Present only for a
+            project that opted into it, and gated like `columns` and the model's
+            SQL: a full column list is the shape of what the author wrote.
+          items:
+            type: object
+            required: [name]
+            properties:
+              name:
+                type: string
+              type:
+                type: string
+                description: The declared type where `schema.yml` gives one, else the inferred one. Omitted when neither is known.
         freshness:
           type: object
           additionalProperties: true

backend/windmill-api/src/jobs.rs

@@ -139,6 +139,7 @@ pub fn workspaced_service() -> Router {
         .route("/run_progress/{id}", get(get_run_progress))
         .route("/run_assets/{id}", get(list_run_assets))
         .route("/dbt_graph/{id}", get(get_dbt_run_graph))
+        .route("/dbt_column_lineage/{id}", get(get_dbt_run_column_lineage))
         .route("/dbt_resumable/{id}", get(get_dbt_resumable))
         .route(
             "/dbt_resumable_script/p/{*script_path}",
@@ -891,21 +892,27 @@ struct AssetProgress {
     error: Option<String>,
 }
 
-/// The asset graph as one run saw it. Pinning to a job needs the full job-read
-/// contract, so it lives on `require_job_read_access` here rather than as a
-/// parameter on `/assets/graph`. See docs/dbt-runtime.md.
-async fn get_dbt_run_graph(
-    authed: ApiAuthed,
-    OptViewToken(view_token): OptViewToken,
-    Extension(db): Extension<DB>,
-    Extension(user_db): Extension<UserDB>,
-    Path((w_id, job_id)): Path<(String, Uuid)>,
-    Query(q): Query<windmill_api_assets::GraphQuery>,
-) -> error::JsonResult<windmill_api_assets::AssetGraphResponse> {
+/// Which project version a dbt view pins to for this job, once the caller has
+/// been shown to be entitled to it.
+///
+/// `Ok(None)` is "answer unpinned", not a refusal: a job that stored no graph of
+/// its own — and one that has aged out of retention — is served the deployed
+/// version rather than an error, so a run page keeps drawing after the run is
+/// gone. Pinning needs the full job-read contract, which is why it lives on
+/// `require_job_read_access` here rather than as a parameter on `/assets/*`.
+/// See docs/dbt-runtime.md.
+async fn dbt_pinned_run(
+    authed: &ApiAuthed,
+    db: &DB,
+    user_db: &UserDB,
+    w_id: &str,
+    job_id: Uuid,
+    view_token: Option<&str>,
+) -> error::Result<Option<windmill_api_assets::PinnedRun>> {
     // The scope domain comes from the URL segment, so `/jobs` asks a scoped token
     // for `jobs:read` alone while the body returned is asset data. Both are
     // required: the job gate below reaches this run, this reaches assets at all.
-    check_scopes(&authed, || "assets:read".to_string())?;
+    check_scopes(authed, || "assets:read".to_string())?;
     let job = sqlx::query!(
         r#"SELECT created_by, runnable_path,
                 CASE WHEN kind = 'script' THEN runnable_id END AS script_hash,
@@ -918,42 +925,69 @@ async fn get_dbt_run_graph(
                            AND g.script_hash IS NULL) AS "editor_graph!"
            FROM v2_job WHERE id = $1 AND workspace_id = $2"#,
         job_id,
-        &w_id
+        w_id
     )
-    .fetch_optional(&db)
+    .fetch_optional(db)
     .await?;
-    // No such job: answer the unpinned graph rather than 404, so a run page whose
-    // job has aged out of retention still draws the deployed version instead of
-    // an error. Reachable only with `assets:read`, which is exactly what
-    // `/assets/graph` would have cost for the same answer.
+    // Unpinned rather than 404 for a job that is gone. Reachable only with
+    // `assets:read`, which is exactly what the unpinned route would have cost
+    // for the same answer.
     let Some(job) = job else {
-        return windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, None).await;
+        return Ok(None);
     };
     require_job_read_access(
-        &db,
-        &user_db,
-        &authed,
-        &w_id,
+        db,
+        user_db,
+        authed,
+        w_id,
         &job_id,
         &job.created_by,
-        view_token.as_deref(),
+        view_token,
     )
     .await?;
     // A preview or flow job names no deployed version, so there is usually no
     // graph to pin to and the workspace one answers. The exception is a job that
     // parsed one itself, which is what the dbt editor's refresh is: its graph
     // belongs to that job alone and nothing else can reach it.
-    let pinned = job
+    Ok(job
         .runnable_path
         .filter(|_| job.script_hash.is_some() || job.editor_graph)
         .map(|path| windmill_api_assets::PinnedRun {
             job_id,
             script_path: path,
             script_hash: job.script_hash,
-        });
+        }))
+}
+
+/// The asset graph as one run saw it.
+async fn get_dbt_run_graph(
+    authed: ApiAuthed,
+    OptViewToken(view_token): OptViewToken,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, job_id)): Path<(String, Uuid)>,
+    Query(q): Query<windmill_api_assets::GraphQuery>,
+) -> error::JsonResult<windmill_api_assets::AssetGraphResponse> {
+    let pinned =
+        dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?;
     windmill_api_assets::asset_graph_for(&authed, &w_id, user_db, db, q, pinned).await
 }
 
+/// One relation's column lineage as one run saw it — the same pin as
+/// `get_dbt_run_graph`, for the trace drawn beside a node of that graph.
+async fn get_dbt_run_column_lineage(
+    authed: ApiAuthed,
+    OptViewToken(view_token): OptViewToken,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, job_id)): Path<(String, Uuid)>,
+    Query(q): Query<windmill_api_assets::ColumnLineageQuery>,
+) -> error::JsonResult<windmill_api_assets::ColumnLineageResponse> {
+    let pinned =
+        dbt_pinned_run(&authed, &db, &user_db, &w_id, job_id, view_token.as_deref()).await?;
+    windmill_api_assets::dbt_column_lineage_for(&authed, &w_id, user_db, q, pinned).await
+}
+
 /// Whether a `dbt retry` submitted by this caller would resume THIS run.
 ///
 /// One failure is saved per script per execution principal, so a page showing an

backend/windmill-common/src/dbt_manifest.rs

@@ -188,6 +188,19 @@ fn graph_digest(ingested: &IngestedManifest, relation_root: &str) -> String {
             .unwrap_or_default()
             .as_bytes(),
     );
+    // Only when there are any, so a project that never asked for the analysis
+    // pass keeps the digest it already has. Hashing an empty section
+    // unconditionally would change every stored digest at once, and every
+    // dynamic run would then store a full snapshot until its script is
+    // redeployed — which reads exactly like the suppression above never working.
+    if !ingested.column_edges.is_empty() {
+        h.update(b"\0");
+        h.update(
+            serde_json::to_string(&ingested.column_edges)
+                .unwrap_or_default()
+                .as_bytes(),
+        );
+    }
     format!("{:x}", h.finalize())
 }
 
@@ -297,6 +310,14 @@ pub async fn prune_dbt_run_graphs(
     )
     .execute(db)
     .await?;
+    sqlx::query!(
+        "DELETE FROM dbt_column_edge
+          WHERE job_id <> '00000000-0000-0000-0000-000000000000'
+            AND ingested_at < now() - make_interval(days => $1)",
+        RUN_GRAPH_RETENTION_DAYS,
+    )
+    .execute(db)
+    .await?;
     // In ONE transaction with the orphan sweep: a restart in the gap leaves graph
     // rows whose marker is gone, and since the sweep runs only when a marker went,
     // every later call computes `retired == 0` and skips them for good.
@@ -325,7 +346,7 @@ pub async fn prune_dbt_run_graphs(
     // partial index here — all of them `WHERE job_id <> DEPLOYED` — and past the
     // keep-count is rare, so the ordinary run should pay for neither.
     if retired > 0 {
-        for table in ["dbt_node", "dbt_edge"] {
+        for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] {
             sqlx::query(&format!(
                 "DELETE FROM {table} t
                   WHERE t.workspace_id = $1 AND t.script_path = $2
@@ -381,6 +402,18 @@ pub struct IngestedNode {
     pub severity: Option<String>,
     pub attached_node: Option<String>,
     pub columns: Option<serde_json::Value>,
+    /// The node's real columns, typed and ordered — `[{"name": …, "type": …}]`,
+    /// from the engine's static analysis. `None` when the project did not ask
+    /// for it or the engine wrote none. Beside `columns` rather than merged into
+    /// it: that one is what the author DECLARED, and stays that.
+    ///
+    /// Skipped when absent, unlike its neighbours, because `graph_digest`
+    /// serializes these nodes: emitting `"column_schema":null` would change
+    /// every stored digest at once, and every dynamic run of a project that
+    /// never asked for the pass would store a full snapshot until its script is
+    /// redeployed.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub column_schema: Option<serde_json::Value>,
     pub freshness: Option<serde_json::Value>,
     /// The transform itself, for the graph to render. The copy taken at
     /// deploy: the file itself is in the script's module bundle.
@@ -388,6 +421,40 @@ pub struct IngestedNode {
     pub original_file_path: Option<String>,
 }
 
+/// One column-to-column edge of the ingested graph.
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Default)]
+#[serde(default)]
+pub struct IngestedColumnEdge {
+    pub parent_unique_id: String,
+    pub parent_column: String,
+    pub child_unique_id: String,
+    pub child_column: String,
+    /// dbt's own word: `copy`, `mod` or `scan`. Kept verbatim — the engine's own
+    /// reader maps those three and passes anything else through, so the set is
+    /// open.
+    pub lineage_kind: String,
+}
+
+/// One column of a node, as the engine's static analysis resolved it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IndexedColumn {
+    pub name: String,
+    /// The declared type where `schema.yml` gives one, else the inferred one.
+    /// Empty when neither is known; the column still belongs to the relation, so
+    /// only the type is left out.
+    pub column_type: String,
+    /// Position in the relation, which is the order the panel lists them in.
+    pub index: i64,
+}
+
+/// What one `--write-index` pass produced: the column edges of the whole
+/// project and the real column schema per node.
+#[derive(Debug, Default)]
+pub struct ColumnIndex {
+    pub edges: Vec<IngestedColumnEdge>,
+    pub columns: HashMap<String, Vec<IndexedColumn>>,
+}
+
 // Serde: an agent worker cannot write these tables directly, so it posts the
 // whole manifest to the server, which stores it with the same function the SQL
 // path uses.
@@ -398,6 +465,9 @@ pub struct IngestedNode {
 pub struct IngestedManifest {
     pub nodes: Vec<IngestedNode>,
     pub edges: Vec<(String, String)>,
+    /// Column-to-column lineage, when the project asked for it and the engine
+    /// produced it. Empty is the normal case — see `attach_column_index`.
+    pub column_edges: Vec<IngestedColumnEdge>,
     /// The `asset` rows the owning script produces (models) and consumes
     /// (sources) — what the lineage graph is drawn from.
     pub assets: Vec<AssetWithAltAccessType>,
@@ -405,6 +475,87 @@ pub struct IngestedManifest {
     pub adapter_type: String,
 }
 
+/// The most column edges one graph stores.
+///
+/// A `scan` edge — the column was read to produce the row, not the value — is
+/// emitted from every join key and every predicate column to every output
+/// column, so one wide model over a multi-column join contributes columns times
+/// predicates edges on its own. The cap is what keeps a project shaped like that
+/// from turning one deploy into a multi-million-row insert; past it the lineage
+/// is truncated and the rest of the graph is unaffected.
+pub const MAX_COLUMN_EDGES: usize = 200_000;
+
+/// Whether the value travelled along this edge, as opposed to the column merely
+/// being read to produce the row.
+///
+/// The graph endpoint serves these two and the trace draws them; `scan` is kept
+/// in the table for a view that wants indirect influence, and is the first thing
+/// `MAX_COLUMN_EDGES` gives up.
+pub fn is_direct(lineage_kind: &str) -> bool {
+    matches!(lineage_kind, "copy" | "mod")
+}
+
+impl IngestedManifest {
+    /// Fold one `--write-index` pass into the graph.
+    ///
+    /// Both halves are scoped to the nodes this graph already kept: the index
+    /// describes the whole project, while the graph describes what this script's
+    /// selection builds plus the parents anchoring its edges, and an edge whose
+    /// endpoint is absent has nothing to draw.
+    pub fn attach_column_index(&mut self, index: ColumnIndex) {
+        let kept: std::collections::HashSet<&str> =
+            self.nodes.iter().map(|n| n.unique_id.as_str()).collect();
+        let mut edges: Vec<IngestedColumnEdge> = index
+            .edges
+            .into_iter()
+            .filter(|e| {
+                kept.contains(e.parent_unique_id.as_str())
+                    && kept.contains(e.child_unique_id.as_str())
+            })
+            .collect();
+        // Sorted and deduplicated for the digest, which decides whether a run
+        // stores a snapshot at all: parquet row order is the engine's and two
+        // passes over one project must not read as two different graphs.
+        //
+        // Direct kinds first, so what the truncation below gives up is `scan` —
+        // the bulk of a wide project's lineage and the kind nothing renders. The
+        // worker's reader already applies this order while decoding, because the
+        // memory bound has to; repeating it here is what makes the ordering a
+        // property of the manifest rather than of one caller's reader, and it is
+        // the only ordering an index assembled some other way would get.
+        edges.sort_by(|a, b| {
+            is_direct(&b.lineage_kind)
+                .cmp(&is_direct(&a.lineage_kind))
+                .then_with(|| a.cmp(b))
+        });
+        edges.dedup();
+        edges.truncate(MAX_COLUMN_EDGES);
+        self.column_edges = edges;
+
+        let mut columns = index.columns;
+        for node in self.nodes.iter_mut() {
+            let Some(mut cols) = columns.remove(&node.unique_id) else {
+                continue;
+            };
+            if cols.is_empty() {
+                continue;
+            }
+            cols.sort_by_key(|c| c.index);
+            node.column_schema = Some(serde_json::Value::Array(
+                cols.into_iter()
+                    // A column the analysis typed as nothing still belongs in
+                    // the list — that it exists is the half `manifest.json`
+                    // could not answer.
+                    .map(|c| match c.column_type.is_empty() {
+                        true => serde_json::json!({ "name": c.name }),
+                        false => serde_json::json!({ "name": c.name, "type": c.column_type }),
+                    })
+                    .collect(),
+            ));
+        }
+    }
+}
+
 /// dbt's `materialized` mapped onto Windmill's write strategy.
 ///
 /// The mapping is exact for the four strategies Windmill has, and deliberately
@@ -655,6 +806,9 @@ pub fn ingest_manifest(
                     .map(|(k, v)| (k.clone(), v.description.clone().unwrap_or_default()))
                     .collect::<BTreeMap<_, _>>())
             }),
+            // Filled by `attach_column_index` when the project asked for it:
+            // the manifest carries declared columns only.
+            column_schema: None,
             freshness: node.freshness.clone(),
             // The transform the graph renders. Capped: a project can hold
             // thousands of models and this is duplicated per deploy, so a
@@ -826,6 +980,16 @@ pub async fn replace_dbt_manifest(
     )
     .execute(&mut **tx)
     .await?;
+    sqlx::query!(
+        "DELETE FROM dbt_column_edge WHERE workspace_id = $1 AND script_path = $2
+           AND script_hash = $3 AND job_id = $4",
+        workspace_id,
+        script_path,
+        script_hash,
+        job_id
+    )
+    .execute(&mut **tx)
+    .await?;
     // The marker, before the rows: a graph with no nodes at all is a legitimate
     // answer for a dynamic run that disabled every model, and the reader must be
     // able to tell it from a run that stored nothing.
@@ -881,7 +1045,7 @@ async fn insert_graph_rows(
             "INSERT INTO dbt_node (workspace_id, script_path, script_hash, job_id, unique_id, \
              resource_type, name, asset_path, materialized, materialize_strategy, unique_key, \
              tags, description, test_kind, test_column, test_args, severity, attached_node, \
-             columns, freshness, raw_code, original_file_path) ",
+             columns, column_schema, freshness, raw_code, original_file_path) ",
         );
         q.push_values(chunk, |mut b, n| {
             b.push_bind(workspace_id)
@@ -903,6 +1067,7 @@ async fn insert_graph_rows(
                 .push_bind(&n.severity)
                 .push_bind(&n.attached_node)
                 .push_bind(&n.columns)
+                .push_bind(&n.column_schema)
                 .push_bind(&n.freshness)
                 .push_bind(&n.raw_code)
                 .push_bind(&n.original_file_path);
@@ -926,6 +1091,26 @@ async fn insert_graph_rows(
         q.push(" ON CONFLICT DO NOTHING");
         q.build().execute(&mut **tx).await?;
     }
+
+    for chunk in ingested.column_edges.chunks(COLUMN_EDGE_INSERT_CHUNK) {
+        let mut q = sqlx::QueryBuilder::new(
+            "INSERT INTO dbt_column_edge (workspace_id, script_path, script_hash, job_id, \
+             parent_unique_id, parent_column, child_unique_id, child_column, lineage_kind) ",
+        );
+        q.push_values(chunk, |mut b, e| {
+            b.push_bind(workspace_id)
+                .push_bind(script_path)
+                .push_bind(script_hash)
+                .push_bind(job_id)
+                .push_bind(&e.parent_unique_id)
+                .push_bind(&e.parent_column)
+                .push_bind(&e.child_unique_id)
+                .push_bind(&e.child_column)
+                .push_bind(&e.lineage_kind);
+        });
+        q.push(" ON CONFLICT DO NOTHING");
+        q.build().execute(&mut **tx).await?;
+    }
     Ok(())
 }
 
@@ -966,7 +1151,7 @@ pub async fn replace_dbt_editor_graph(
 ) -> Result<()> {
     // By job alone, so re-executing one — a zombie recovered onto another
     // worker — replaces its rows rather than colliding with them.
-    for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] {
+    for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] {
         sqlx::query(&format!(
             "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = $2 AND script_hash IS NULL"
         ))
@@ -1016,7 +1201,7 @@ pub async fn replace_dbt_editor_graph(
     .fetch_all(&mut **tx)
     .await?;
     if !retired.is_empty() {
-        for table in ["dbt_node", "dbt_edge"] {
+        for table in ["dbt_node", "dbt_edge", "dbt_column_edge"] {
             sqlx::query(&format!(
                 "DELETE FROM {table} WHERE workspace_id = $1 AND job_id = ANY($2) \
                    AND script_hash IS NULL"
@@ -1035,6 +1220,8 @@ pub async fn replace_dbt_editor_graph(
 const NODE_INSERT_CHUNK: usize = 2000;
 /// Six columns, so the same ceiling allows far more.
 const EDGE_INSERT_CHUNK: usize = 8000;
+/// Nine columns, and by far the most numerous rows of the three.
+const COLUMN_EDGE_INSERT_CHUNK: usize = 6000;
 
 /// Clear one VERSION's graph: the delete-by-hash route, which only soft-deletes
 /// its `script` row and so fires no cascade, and the ingest that finds no
@@ -1070,6 +1257,15 @@ pub async fn clear_dbt_manifest_version(
     )
     .execute(&mut **tx)
     .await?;
+    sqlx::query!(
+        "DELETE FROM dbt_column_edge
+          WHERE workspace_id = $1 AND script_path = $2 AND script_hash = $3",
+        workspace_id,
+        script_path,
+        script_hash
+    )
+    .execute(&mut **tx)
+    .await?;
     // The marker too, and every job's: a marker left standing for rows that are
     // gone is read as a snapshot, and its digest still answers the suppression
     // check — so an identical run would write nothing and then render an empty
@@ -1105,7 +1301,7 @@ pub async fn clear_dbt_editor_graphs(
     workspace_id: &str,
     script_path: &str,
 ) -> Result<()> {
-    for table in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] {
+    for table in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] {
         sqlx::query(&format!(
             "DELETE FROM {table}
               WHERE workspace_id = $1 AND script_path = $2 AND script_hash IS NULL"
@@ -1765,6 +1961,69 @@ mod tests {
         assert_eq!(back.assets.len(), ingested.assets.len());
         assert_eq!(back.assets[0].path, ingested.assets[0].path);
     }
+
+    // The index describes the whole PROJECT while the graph describes what this
+    // script's selection builds, so an edge whose endpoint the graph does not
+    // hold has nothing to draw and must not be stored.
+    #[test]
+    fn column_lineage_is_scoped_to_the_nodes_the_graph_kept() {
+        let mut i = ingested();
+        let kept = "model.jaffle_shop.customers";
+        let dropped = "model.other_project.elsewhere";
+        i.attach_column_index(ColumnIndex {
+            edges: vec![
+                edge("model.jaffle_shop.orders_daily", "id", kept, "id", "copy"),
+                edge(dropped, "id", kept, "id", "copy"),
+                edge(kept, "id", dropped, "id", "copy"),
+            ],
+            columns: [
+                (
+                    kept.to_string(),
+                    vec![
+                        col("total", "Float64", 1),
+                        col("id", "Int32", 0),
+                        col("untyped", "", 2),
+                    ],
+                ),
+                (dropped.to_string(), vec![col("id", "Int32", 0)]),
+            ]
+            .into(),
+        });
+        assert_eq!(
+            i.column_edges
+                .iter()
+                .map(|e| (e.parent_unique_id.as_str(), e.child_unique_id.as_str()))
+                .collect::<Vec<_>>(),
+            vec![("model.jaffle_shop.orders_daily", kept)]
+        );
+        // In `column_index` order, and a column the analysis could not type still
+        // belongs to the relation.
+        assert_eq!(
+            node(&i, kept).column_schema,
+            Some(serde_json::json!([
+                {"name": "id", "type": "Int32"},
+                {"name": "total", "type": "Float64"},
+                {"name": "untyped"},
+            ]))
+        );
+        assert!(node(&i, "model.jaffle_shop.orders_daily")
+            .column_schema
+            .is_none());
+    }
+
+    fn edge(from: &str, from_col: &str, to: &str, to_col: &str, kind: &str) -> IngestedColumnEdge {
+        IngestedColumnEdge {
+            parent_unique_id: from.into(),
+            parent_column: from_col.into(),
+            child_unique_id: to.into(),
+            child_column: to_col.into(),
+            lineage_kind: kind.into(),
+        }
+    }
+
+    fn col(name: &str, column_type: &str, index: i64) -> IndexedColumn {
+        IndexedColumn { name: name.into(), column_type: column_type.into(), index }
+    }
 }
 
 /// Record one model's state for THIS RUN.

backend/windmill-common/tests/dbt_graph_storage.rs

@@ -8,8 +8,8 @@
 use sqlx::{Pool, Postgres};
 use windmill_common::dbt_manifest::{
     clear_dbt_editor_graphs, clear_dbt_manifest_version, prune_dbt_run_graphs,
-    replace_dbt_editor_graph, replace_dbt_manifest, IngestedManifest, IngestedNode,
-    DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT,
+    replace_dbt_editor_graph, replace_dbt_manifest, IngestedColumnEdge, IngestedManifest,
+    IngestedNode, DBT_EDITOR_GRAPHS_KEPT, DEPLOYED_GRAPH, DEPLOYED_GRAPH_VERSIONS_KEPT,
 };
 
 const WS: &str = "test-workspace";
@@ -52,10 +52,36 @@ fn manifest(names: &[&str]) -> IngestedManifest {
             .windows(2)
             .map(|w| (format!("model.p.{}", w[0]), format!("model.p.{}", w[1])))
             .collect(),
+        // Same reason: a project that opted into the analysis pass has these, and
+        // a fixture without them leaves every column-edge insert and sweep in
+        // this file unexecuted.
+        column_edges: names
+            .windows(2)
+            .map(|w| IngestedColumnEdge {
+                parent_unique_id: format!("model.p.{}", w[0]),
+                parent_column: w[0].to_string(),
+                child_unique_id: format!("model.p.{}", w[1]),
+                child_column: w[1].to_string(),
+                lineage_kind: "copy".to_string(),
+            })
+            .collect(),
         ..Default::default()
     }
 }
 
+/// Column edges of one version, so the sweeps can be shown to reach them.
+async fn column_edges_for(db: &Pool<Postgres>, hash: i64) -> i64 {
+    sqlx::query_scalar!(
+        "SELECT count(*) FROM dbt_column_edge WHERE workspace_id = $1 AND script_hash = $2",
+        WS,
+        hash
+    )
+    .fetch_one(db)
+    .await
+    .unwrap()
+    .unwrap_or(0)
+}
+
 /// Edges for one version, so a test can assert the batched insert ran at all.
 async fn edges_for(db: &Pool<Postgres>, hash: i64) -> i64 {
     sqlx::query_scalar!(
@@ -275,6 +301,8 @@ async fn deleting_the_script_cascades_to_every_sidecar(db: Pool<Postgres>) {
     assert_eq!(nodes_for(&db, 2, DEPLOYED_GRAPH).await, 0);
     assert_eq!(edges_for(&db, 1).await, 0);
     assert_eq!(edges_for(&db, 2).await, 0);
+    assert_eq!(column_edges_for(&db, 1).await, 0);
+    assert_eq!(column_edges_for(&db, 2).await, 0);
     assert_eq!(markers_for_path(&db).await, 0);
 }
 
@@ -315,7 +343,7 @@ async fn the_sweep_takes_old_snapshots_and_spares_the_version(db: Pool<Postgres>
     tx.commit().await.unwrap();
 
     // Age one snapshot past the window, rows and marker together.
-    for t in ["dbt_node", "dbt_edge", "dbt_graph_snapshot"] {
+    for t in ["dbt_node", "dbt_edge", "dbt_column_edge", "dbt_graph_snapshot"] {
         sqlx::query(&format!(
             "UPDATE {t} SET ingested_at = now() - interval '400 days' WHERE job_id = $1"
         ))

backend/windmill-worker/Cargo.toml

@@ -114,6 +114,10 @@ rust_decimal.workspace = true
 jsonwebtoken.workspace = true
 sha2.workspace = true
 hmac.workspace = true
+# Reads the dbt engine's `target/index/*.parquet`, which is where column-level
+# lineage lives. Unconditional rather than behind the `parquet` FEATURE: that one
+# is object storage, and a build without it still runs dbt jobs.
+parquet.workspace = true
 pem = { workspace = true, optional = true }
 rsa = { workspace = true, optional = true }
 urlencoding.workspace = true

backend/windmill-worker/src/dbt_column_index.rs

@@ -0,0 +1,508 @@
+//! Column-level lineage and real column schemas, from the engine's own static
+//! analysis.
+//!
+//! `manifest.json` carries neither. What does is the parquet index an engine
+//! writes under `dbt compile --static-analysis strict --write-index`:
+//! `dbt.column_lineage.parquet` (column-to-column edges, each labelled `copy`,
+//! `mod` or `scan`) and `dbt.node_columns.parquet` (every column of every node,
+//! typed and ordered, rather than only the ones an author documented).
+//!
+//! Three properties shape everything here, all of them measured against the real
+//! engines rather than assumed:
+//!
+//! - **Strict analysis rejects SQL the default accepts.** An unresolvable
+//!   identifier is an error under `strict` and compiles fine otherwise, so this
+//!   is a SEPARATE pass with its own `--target-path`, never a flag on the build,
+//!   and it is opt-in per project.
+//! - **A failed pass still writes the index**, holding every edge of the models
+//!   that did analyze. So the artifact is read whatever the exit status.
+//! - **The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 accepts
+//!   `--write-index`, declares the views over these two tables in its own
+//!   `views.sql`, and writes neither file; only Fusion does today. Nothing here
+//!   asks which engine it is beyond "has the flag" — a release that starts
+//!   writing them is picked up with no change.
+
+use std::collections::HashSet;
+use std::ops::ControlFlow;
+use std::path::Path;
+use std::time::Duration;
+
+use parquet::file::reader::{FileReader, SerializedFileReader};
+use parquet::record::{Field, Row};
+use uuid::Uuid;
+use windmill_common::dbt_manifest::{
+    is_direct, ColumnIndex, IndexedColumn, IngestedColumnEdge, MAX_COLUMN_EDGES,
+};
+use windmill_common::error;
+use windmill_common::worker::Connection;
+use windmill_parser_yaml::dbt::DbtDescriptor;
+use windmill_queue::append_logs;
+
+use crate::dbt_executor::{dbt_command, Invocation, PreparedProject};
+use crate::handle_child::JobCtx;
+
+/// Where the lineage pass writes, relative to the project directory.
+///
+/// Its own tree, not the runtime's `wm_target`: a `dbt compile` writes
+/// `manifest.json` and `run_results.json` like any other invocation, and after a
+/// build those two are what the graph ingest and `dbt retry` read.
+const CLL_ARTIFACTS_DIR: &str = "wm_target_cll";
+
+const COLUMN_LINEAGE_PARQUET: &str = "dbt.column_lineage.parquet";
+const NODE_COLUMNS_PARQUET: &str = "dbt.node_columns.parquet";
+
+/// Run the lineage pass and read what it produced.
+///
+/// Two steps with deliberately different contracts, because conflating them is
+/// what made a best-effort annotation able to fail the job it annotates:
+///
+/// - [`compile_index`] runs a subprocess and owns the JOB's semantics. Only a
+///   cancellation or the job's own deadline can `Err` out of it; a non-zero exit
+///   and an over-long output are outcomes, not failures.
+/// - [`read_index`] owns the ARTIFACT's semantics. It is infallible and
+///   memory-bounded, and knows nothing about the job.
+///
+/// The phase budget wraps the compile alone. A budget around the whole pass
+/// would time out with a decode still running on a blocking thread, which is
+/// precisely what "the build below gets the rest" must not mean.
+pub(crate) async fn collect(
+    p: &PreparedProject,
+    descriptor: &DbtDescriptor,
+    inv: &Invocation,
+    ctx: &mut JobCtx<'_>,
+    job_id: &Uuid,
+    w_id: &str,
+    conn: &Connection,
+    kept: &HashSet<&str>,
+) -> error::Result<Option<ColumnIndex>> {
+    if !descriptor.column_lineage {
+        return Ok(None);
+    }
+    if !p.engine.engine.writes_column_index() {
+        append_logs(
+            job_id,
+            w_id,
+            format!(
+                "\n`column_lineage` is set, but the {} engine has no `--write-index`: column \
+                 lineage needs an engine that does static analysis. The rest of the graph is \
+                 unaffected.\n",
+                p.engine.engine.as_str()
+            ),
+            conn,
+        )
+        .await;
+        return Ok(None);
+    }
+
+    let index_dir = p.project_dir.join(CLL_ARTIFACTS_DIR).join("index");
+    let Some(compiled) = compile_index(p, descriptor, inv, ctx, job_id, w_id, conn).await? else {
+        return Ok(None);
+    };
+    let coverage = Coverage::of(&compiled);
+
+    // What only the pass knows. The COUNTS are logged where the index is folded
+    // into the graph, since the graph decides how much of it is kept.
+    let note = match read_index(&index_dir, kept).await {
+        Artifact::Read(index) => {
+            if let Some(note) = coverage.caveat() {
+                log(job_id, w_id, note, &compiled.stderr, conn).await;
+            }
+            return Ok(Some(index));
+        }
+        // Said apart from the one above, because it sends the reader somewhere
+        // else entirely: the engine did its job and this runtime could not read
+        // what it wrote.
+        Artifact::Unreadable(why) => format!(
+            "No column lineage: `{COLUMN_LINEAGE_PARQUET}` was written but could not be read \
+             ({why}). The graph is unaffected."
+        ),
+        Artifact::Missing => format!(
+            "No column lineage: the analysis pass wrote no `{COLUMN_LINEAGE_PARQUET}`. Only an \
+             engine that computes it does, and only for the warehouses it analyzes natively — \
+             the flag alone is not the capability."
+        ),
+    };
+    // The engine's own diagnostics come along. They are how a reader learns that
+    // this adapter turned static analysis off, which it reports as a warning on
+    // a SUCCESSFUL compile that nothing else would show.
+    log(job_id, w_id, &note, &compiled.stderr, conn).await;
+    Ok(None)
+}
+
+async fn log(job_id: &Uuid, w_id: &str, note: &str, stderr: &str, conn: &Connection) {
+    append_logs(
+        job_id,
+        w_id,
+        format!("\n{note}\n{}", diagnostics(stderr)),
+        conn,
+    )
+    .await;
+}
+
+/// How completely the analysis compile covered the project.
+///
+/// Every way the COMPILE can disappoint is a value here rather than an error. An
+/// `Err` from `compile_index` is the JOB's — a cancellation or its deadline —
+/// and must fail it; the pass giving up on its own terms is `Ok(None)` and has
+/// already been logged.
+enum Coverage {
+    /// Every model analyzed.
+    Whole,
+    /// `--static-analysis strict` rejected part of the project. Whatever it did
+    /// analyze is still in the index.
+    Partial,
+    /// The output ceiling killed the compile mid-run. Distinct from `Partial`:
+    /// nothing rejected the project, but the index is however far it had got, so
+    /// it is not `Whole` either.
+    Truncated,
+}
+
+impl Coverage {
+    fn of(c: &crate::dbt_executor::Captured) -> Self {
+        match (c.truncated, c.success) {
+            (true, _) => Coverage::Truncated,
+            (false, true) => Coverage::Whole,
+            (false, false) => Coverage::Partial,
+        }
+    }
+
+    /// What to tell the reader when an index WAS produced. `None` for a run that
+    /// covered everything, which needs no caveat.
+    fn caveat(&self) -> Option<&'static str> {
+        match self {
+            Coverage::Whole => None,
+            Coverage::Partial => Some(
+                "Column lineage: `--static-analysis strict` rejected part of the project, so \
+                 the lineage covers only the models it could analyze.",
+            ),
+            Coverage::Truncated => Some(
+                "Column lineage: the analysis pass printed more than this runtime reads and was \
+                 stopped, so the lineage covers only the models it had reached.",
+            ),
+        }
+    }
+}
+
+/// Run `dbt compile --static-analysis strict --write-index`, under this phase's
+/// share of the job's clock.
+///
+/// `Ok(None)` is "the pass gave up and said so"; `Err` is the job's own
+/// cancellation or deadline and must propagate. Nothing outlives this function:
+/// the budget is a race around the child, and dropping that future kills it
+/// through `run_captured`'s `kill_on_drop`.
+async fn compile_index(
+    p: &PreparedProject,
+    descriptor: &DbtDescriptor,
+    inv: &Invocation,
+    ctx: &mut JobCtx<'_>,
+    job_id: &Uuid,
+    w_id: &str,
+    conn: &Connection,
+) -> error::Result<Option<crate::dbt_executor::Captured>> {
+    // A previous pass in the same job directory — a retry's second attempt —
+    // would otherwise be read back as this one's answer.
+    tokio::fs::remove_dir_all(p.project_dir.join(CLL_ARTIFACTS_DIR))
+        .await
+        .ok();
+
+    let mut cmd = dbt_command(
+        p,
+        &[
+            "compile",
+            "--static-analysis",
+            "strict",
+            "--write-index",
+            // Documented as what builds the CLL graph, and `--write-index` alone
+            // happens to imply it on the engine probed. Passed explicitly so the
+            // pass does not depend on which of the two is doing the work.
+            "--write-lineage",
+            "--target-path",
+            CLL_ARTIFACTS_DIR,
+        ],
+    );
+    // The flag already wins over the env var dbt_command sets, but setting both
+    // means this pass cannot write into the runtime's artifacts even if that
+    // precedence ever changes — and what is in there after a build is the
+    // `run_results.json` a `dbt retry` resumes from.
+    cmd.env("DBT_TARGET_PATH", CLL_ARTIFACTS_DIR);
+    crate::dbt_executor::add_vars(&mut cmd, descriptor, inv)?;
+    // Captured rather than streamed: a strict-analysis failure is a wall of
+    // diagnostics about SQL the build itself accepts, and this pass decides
+    // nothing about whether that build runs.
+    // Read before the future below borrows `ctx` mutably.
+    let budget = phase_budget(ctx);
+    let run = crate::dbt_executor::run_captured(
+        cmd,
+        "dbt compile (column lineage)",
+        ctx,
+        job_id,
+        w_id,
+        conn,
+        CLL_MAX_OUTPUT_BYTES,
+        // The ceiling is this pass's, not the job's: a compile that prints more
+        // than it than we care to read has still analyzed the project, and the
+        // index it wrote is on disk either way.
+        crate::dbt_executor::Overflow::Truncate,
+    );
+    let Some(budget) = budget else {
+        return Ok(Some(run.await?));
+    };
+    match tokio::time::timeout(budget, run).await {
+        Ok(r) => Ok(Some(r?)),
+        Err(_) => {
+            append_logs(
+                job_id,
+                w_id,
+                format!(
+                    "\nNo column lineage: the analysis pass did not finish within {}s, half of \
+                     what was left of this job's time. The build below gets the rest.\n",
+                    budget.as_secs()
+                ),
+                conn,
+            )
+            .await;
+            Ok(None)
+        }
+    }
+}
+
+/// stdout the pass may produce. It is a compile, so this is diagnostics rather
+/// than data.
+const CLL_MAX_OUTPUT_BYTES: usize = 1 << 20;
+
+/// The share of the job's remaining wall clock this pass may spend.
+///
+/// A per-run refresh ingests BEFORE the build and shares the job's one deadline,
+/// so an unbounded pass on a slow project would hand `dbt build` an expired
+/// budget and fail the run it exists only to annotate. Half leaves the build at
+/// least as long as the annotation was allowed to take.
+///
+/// Spent as a race around the COMPILE rather than as a shortened deadline handed
+/// to the runner: the runner reports its expiry as an `Err`, indistinguishable
+/// from a cancellation or the job's own deadline, and those two MUST fail the
+/// job. Expiring here is this budget and nothing else. The child dies with the
+/// dropped future through `run_captured`'s `kill_on_drop`, and the decode is
+/// outside the race so nothing survives it.
+fn phase_budget(ctx: &JobCtx<'_>) -> Option<Duration> {
+    ctx.timeout()
+        .map(|left| Duration::from_secs((left.max(0) as u64 / 2).max(1)))
+}
+
+/// The tail of what the engine said, bounded. The whole of it is every rendered
+/// model on a large project, which is not what a job log is for.
+const DIAGNOSTIC_LINES: usize = 40;
+
+fn diagnostics(out: &str) -> String {
+    let lines: Vec<&str> = out.lines().filter(|l| !l.trim().is_empty()).collect();
+    let tail = &lines[lines.len().saturating_sub(DIAGNOSTIC_LINES)..];
+    match tail.is_empty() {
+        true => String::new(),
+        false => format!("{}\n", tail.join("\n")),
+    }
+}
+
+/// What came back from the artifact. Never an `Err`: this half owns none of the
+/// job's semantics. `Unreadable` is separate from `Missing` because the two send
+/// a reader looking in different places — one at their engine and adapter, the
+/// other at a file that exists.
+enum Artifact {
+    Read(ColumnIndex),
+    Missing,
+    Unreadable(String),
+}
+
+/// Read both parquets, if the lineage one is there.
+///
+/// The column schemas alone are not worth a graph: they arrive with the lineage
+/// or not at all, and a node's declared columns already answer for the case
+/// where the pass never ran.
+async fn read_index(index_dir: &Path, kept: &HashSet<&str>) -> Artifact {
+    let lineage = index_dir.join(COLUMN_LINEAGE_PARQUET);
+    if !tokio::fs::try_exists(&lineage).await.unwrap_or(false) {
+        return Artifact::Missing;
+    }
+    let columns = index_dir.join(NODE_COLUMNS_PARQUET);
+    // Owned, because the decode moves to a blocking thread. The index describes
+    // the whole project while this graph describes one selection of it, so
+    // scoping HERE is what keeps the bound below from being spent on rows the
+    // graph would discard anyway.
+    let kept: HashSet<String> = kept.iter().map(|s| (*s).to_string()).collect();
+    // Decompressing and decoding a parquet is CPU work on a file the engine just
+    // wrote, so it does not belong on the runtime's poll thread.
+    let read =
+        tokio::task::spawn_blocking(move || read_index_blocking(&lineage, &columns, &kept)).await;
+    match read {
+        Ok(Ok(index)) => Artifact::Read(index),
+        Ok(Err(e)) => Artifact::Unreadable(e.to_string()),
+        Err(e) => Artifact::Unreadable(e.to_string()),
+    }
+}
+
+fn read_index_blocking(
+    lineage: &Path,
+    columns: &Path,
+    kept: &HashSet<String>,
+) -> error::Result<ColumnIndex> {
+    let mut out = ColumnIndex::default();
+    // ONE pass, with the two kinds bucketed as they arrive. Direct edges are
+    // what a trace draws, so they get the whole budget; `scan` — the bulk of a
+    // wide project's index and the kind nothing renders — fills only what is
+    // left over at the end. Reading the file twice to get that ordering would
+    // double the decode of exactly the large index this bound exists for.
+    let mut scan: Vec<IngestedColumnEdge> = Vec::new();
+    for_each_row(lineage, |row| {
+        let lineage_kind = string(row, "lineage_kind");
+        let parent_unique_id = string(row, "from_node_unique_id");
+        let child_unique_id = string(row, "to_node_unique_id");
+        let parent_column = string(row, "from_column_name");
+        let child_column = string(row, "to_column_name");
+        // A column of a node the analysis could not name is not an endpoint the
+        // graph can draw, and neither is one outside this graph's nodes.
+        if parent_column.is_empty()
+            || child_column.is_empty()
+            || !kept.contains(&parent_unique_id)
+            || !kept.contains(&child_unique_id)
+        {
+            return ControlFlow::Continue(());
+        }
+        let edge = IngestedColumnEdge {
+            parent_unique_id,
+            parent_column,
+            child_unique_id,
+            child_column,
+            lineage_kind,
+        };
+        // The bound covers BOTH buckets, so the pass never holds more than one
+        // budget's worth however the kinds are distributed.
+        let held = out.edges.len() + scan.len();
+        if is_direct(&edge.lineage_kind) {
+            // A direct edge displaces a `scan` one: the budget exists to be
+            // spent on what a trace draws.
+            if held >= MAX_COLUMN_EDGES {
+                scan.pop();
+            }
+            out.edges.push(edge);
+            // The edge that FILLS the budget ends the read, not the next one to
+            // arrive: once the displacing kind is full nothing later in the file
+            // can be kept, and waiting for another direct edge to say so decodes
+            // a `scan`-only tail all the way to the backstop for nothing.
+            return match out.edges.len() >= MAX_COLUMN_EDGES {
+                true => ControlFlow::Break(()),
+                false => ControlFlow::Continue(()),
+            };
+        }
+        if held < MAX_COLUMN_EDGES {
+            scan.push(edge);
+        }
+        // Not a stopping point even when full: a direct edge still to come takes
+        // a `scan` entry's place.
+        ControlFlow::Continue(())
+    })?;
+    out.edges.append(&mut scan);
+    // Absent is normal — an engine can write the lineage table and not this one —
+    // and unreadable is not worth losing the lineage over.
+    let mut held = 0usize;
+    let _ = for_each_row(columns, |row| {
+        let unique_id = string(row, "unique_id");
+        let name = string(row, "column_name");
+        if held >= MAX_INDEXED_COLUMNS {
+            return ControlFlow::Break(());
+        }
+        if name.is_empty() || !kept.contains(&unique_id) {
+            return ControlFlow::Continue(());
+        }
+        held += 1;
+        // The author's `data_type` where `schema.yml` gives one, since that is
+        // what the project calls the column; the analysis's own inference
+        // otherwise.
+        let column_type = match string(row, "declared_type") {
+            t if !t.is_empty() => t,
+            _ => string(row, "inferred_type"),
+        };
+        out.columns
+            .entry(unique_id)
+            .or_default()
+            .push(IndexedColumn {
+                name,
+                column_type,
+                index: int(row, "column_index").unwrap_or(i64::MAX),
+            });
+        ControlFlow::Continue(())
+    });
+    Ok(out)
+}
+
+/// The most rows of `dbt.node_columns.parquet` one pass keeps. One per column of
+/// the project, so the same bound as the edges is far more than any project
+/// reaches; it exists for the same reason.
+const MAX_INDEXED_COLUMNS: usize = MAX_COLUMN_EDGES;
+
+/// The most rows of an index one pass DECODES, whatever it keeps of them.
+///
+/// A bound on work rather than on memory, and the two are separate because the
+/// input this defends against is the one that cannot be collected: `scan`
+/// lineage is emitted from every predicate and join column to every output
+/// column, so a project shaped that way writes an index whose row count is
+/// quadratic in its widest model. This pass runs outside the phase budget, on a
+/// blocking thread, and its whole contract is that it cannot fail a deploy or a
+/// run — so the file it walks needs an end even when almost nothing in it is
+/// retained.
+const MAX_INDEX_ROWS: usize = 4_000_000;
+
+/// Decode a parquet a row at a time, handing each to `f` and never holding two.
+///
+/// Collecting first would put a `Vec<Row>` — each row carrying its own copy of
+/// every column NAME — in front of the caller's own bound, which is what would
+/// take the worker process down on the index described above.
+///
+/// `f` says when it has all it will take, and that is the ordinary end: this
+/// runs outside the phase budget, so every row decoded past the point of being
+/// able to keep one is wall clock the build below does not get.
+fn for_each_row(path: &Path, mut f: impl FnMut(&Row) -> ControlFlow<()>) -> error::Result<()> {
+    let fail = |e: parquet::errors::ParquetError| {
+        error::Error::internal_err(format!("reading {}: {e}", path.display()))
+    };
+    let file = std::fs::File::open(path)
+        .map_err(|e| error::Error::internal_err(format!("opening {}: {e}", path.display())))?;
+    let reader = SerializedFileReader::new(file).map_err(fail)?;
+    for (n, row) in reader.get_row_iter(None).map_err(fail)?.enumerate() {
+        if n >= MAX_INDEX_ROWS {
+            tracing::warn!(
+                "dbt column index: {} holds more than {MAX_INDEX_ROWS} rows; the rest is dropped",
+                path.display()
+            );
+            break;
+        }
+        if f(&row.map_err(fail)?).is_break() {
+            break;
+        }
+    }
+    Ok(())
+}
+
+/// By NAME, not by position: these tables are the engine's own schema and it
+/// adds columns to them between releases.
+fn field<'a>(row: &'a Row, name: &str) -> Option<&'a Field> {
+    row.get_column_iter()
+        .find(|(k, _)| k.as_str() == name)
+        .map(|(_, v)| v)
+}
+
+fn string(row: &Row, name: &str) -> String {
+    match field(row, name) {
+        Some(Field::Str(s)) => s.clone(),
+        Some(Field::Bytes(b)) => String::from_utf8_lossy(b.data()).into_owned(),
+        _ => String::new(),
+    }
+}
+
+fn int(row: &Row, name: &str) -> Option<i64> {
+    match field(row, name) {
+        Some(Field::Long(v)) => Some(*v),
+        Some(Field::Int(v)) => Some(*v as i64),
+        Some(Field::Short(v)) => Some(*v as i64),
+        Some(Field::UInt(v)) => Some(*v as i64),
+        Some(Field::ULong(v)) => i64::try_from(*v).ok(),
+        _ => None,
+    }
+}

backend/windmill-worker/src/dbt_executor.rs

@@ -653,12 +653,23 @@ pub(crate) async fn dbt_dep(
         None => GraphPublisher::Unversioned,
     };
     let superseded = if let Some(warehouse) = prepared.warehouse.as_deref() {
-        let ingested = windmill_common::dbt_manifest::ingest_manifest(
+        let mut ingested = windmill_common::dbt_manifest::ingest_manifest(
             &manifest,
             warehouse,
             prepared.default_database.as_deref(),
             selected.as_ref(),
         );
+        attach_column_index(
+            &mut ingested,
+            &prepared,
+            &descriptor,
+            &inv,
+            &mut ctx,
+            job_id,
+            w_id,
+            &conn,
+        )
+        .await?;
         let published = persist_ingest(
             db,
             w_id,
@@ -2882,7 +2893,8 @@ async fn run_show(
         conn,
         SHOW_MAX_OUTPUT_BYTES,
     )
-    .await?;
+    .await?
+    .stdout;
     // dbt frames the rows as `{"node": …, "show": [ … ]}`, pretty-printed, with a
     // banner before and a deprecation summary after — so neither "the line starting
     // with `{`" nor "first `{` to the end" parses. A streaming deserializer stops at
@@ -3064,12 +3076,23 @@ async fn run_parse_only(
     // manifest and the selection while the warehouse only keys them — so a project
     // with no warehouse identity still reports what dbt found. The placeholder
     // reaches no row: the guard below returns before anything is written.
-    let ingested = windmill_common::dbt_manifest::ingest_manifest(
+    let mut ingested = windmill_common::dbt_manifest::ingest_manifest(
         &manifest,
         p.warehouse.as_deref().unwrap_or("unkeyed"),
         p.default_database.as_deref(),
         selected.as_ref(),
     );
+    attach_column_index(
+        &mut ingested,
+        p,
+        descriptor,
+        inv,
+        ctx,
+        &job.id,
+        &job.workspace_id,
+        conn,
+    )
+    .await?;
     result.nodes = ingested.nodes.len();
     result.edges = ingested.edges.len();
     for n in &ingested.nodes {
@@ -3143,6 +3166,57 @@ async fn run_parse_only(
     Ok(to_raw_value(&result))
 }
 
+/// Fold this project's column lineage into the graph about to be stored, when
+/// the descriptor asked for it.
+///
+/// One helper for all three ingests — deploy, editor parse, per-run refresh —
+/// because a graph that carries column lineage in one provenance and not another
+/// reads as the lineage having disappeared.
+async fn attach_column_index(
+    ingested: &mut windmill_common::dbt_manifest::IngestedManifest,
+    p: &PreparedProject,
+    descriptor: &DbtDescriptor,
+    inv: &Invocation,
+    ctx: &mut JobCtx<'_>,
+    job_id: &Uuid,
+    w_id: &str,
+    conn: &Connection,
+) -> error::Result<()> {
+    // The nodes this graph kept, so the pass reads only rows it could store: the
+    // index describes the whole project, this graph one selection of it.
+    let kept: std::collections::HashSet<&str> =
+        ingested.nodes.iter().map(|n| n.unique_id.as_str()).collect();
+    let index =
+        crate::dbt_column_index::collect(p, descriptor, inv, ctx, job_id, w_id, conn, &kept).await?;
+    drop(kept);
+    let Some(index) = index else {
+        return Ok(());
+    };
+    let found = index.edges.len();
+    ingested.attach_column_index(index);
+    let kept = ingested.column_edges.len();
+    let typed: usize = ingested
+        .nodes
+        .iter()
+        .filter(|n| n.column_schema.is_some())
+        .count();
+    // Counted here rather than at the pass: the index describes the whole
+    // project and this graph describes one selection of it, so `found` is what
+    // dbt produced and `kept` is what the graph can draw.
+    let dropped = match found.saturating_sub(kept) {
+        0 => String::new(),
+        n => format!(" ({n} outside this graph or past the cap)"),
+    };
+    append_logs(
+        job_id,
+        w_id,
+        format!("\nIngested {kept} column lineage edges{dropped} and typed {typed} nodes\n"),
+        conn,
+    )
+    .await;
+    Ok(())
+}
+
 /// Refresh the stored graph from the manifest this run produced.
 async fn ingest_from_run(
     p: &PreparedProject,
@@ -3164,12 +3238,23 @@ async fn ingest_from_run(
     // filter this run's manifest by a different node set than it built.
     let selected =
         resolve_selection(p, descriptor, inv, ctx, &job.id, &job.workspace_id, conn).await?;
-    let ingested = windmill_common::dbt_manifest::ingest_manifest(
+    let mut ingested = windmill_common::dbt_manifest::ingest_manifest(
         &manifest,
         warehouse,
         p.default_database.as_deref(),
         selected.as_ref(),
     );
+    attach_column_index(
+        &mut ingested,
+        p,
+        descriptor,
+        inv,
+        ctx,
+        &job.id,
+        &job.workspace_id,
+        conn,
+    )
+    .await?;
     // Only a run whose models are its own snapshots per run. A static
     // descriptor at a moved profile re-ingests the VERSION's graph, since the
     // move outlives the run; one that neither drifted nor overrode anything
@@ -3428,7 +3513,9 @@ async fn resolve_selection(
     // through the job-log writer, which `NO_LOGS_AT_ALL` discards — the selection
     // would resolve to the empty set and the ingest would wipe the script's assets
     // while dbt went on building the descriptor's models.
-    let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES).await?;
+    let stdout = run_capturing(cmd, "dbt ls", ctx, job_id, w_id, conn, LS_MAX_OUTPUT_BYTES)
+        .await?
+        .stdout;
     let mut set = std::collections::HashSet::new();
     for line in stdout.lines() {
         let line = line.trim();
@@ -3459,6 +3546,34 @@ async fn resolve_selection(
 /// what is kept is the TAIL, because dbt prints its error summary last.
 const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024;
 
+/// What a captured invocation produced. `stderr` is where dbt writes its
+/// diagnostics — the errors and warnings block — so a caller that has to explain
+/// a SUCCESSFUL run needs it as much as a failing one does.
+pub(crate) struct Captured {
+    pub stdout: String,
+    pub stderr: String,
+    /// Whether the child exited zero. Separate from the `Result` on purpose: an
+    /// `Err` from `run_captured` is the JOB's — a cancellation or its deadline —
+    /// so a caller that tolerates a failed command must still propagate one.
+    pub success: bool,
+    /// Whether the output ceiling cut the child short. Only ever true under
+    /// [`Overflow::Truncate`].
+    pub truncated: bool,
+}
+
+/// What an over-long stdout means to the caller.
+///
+/// The ceiling belongs to the PASS, not to the job: a caller that only annotates
+/// a job wants to keep what it read and carry on, while one whose whole result
+/// is that output has nothing to return without it.
+#[derive(PartialEq, Eq, Clone, Copy)]
+pub(crate) enum Overflow {
+    /// Fail the job. For a command whose output IS the answer.
+    Fail,
+    /// Stop reading, kill the child, and report `truncated`.
+    Truncate,
+}
+
 /// Run a command for its stdout under the job's cancellation and timeout.
 /// The same poller `handle_child` uses drives them, so a cancel or a deadline
 /// drops the wait future — which owns the child, and `kill_on_drop` then
@@ -3471,7 +3586,7 @@ const CAPTURE_MAX_STDERR_BYTES: usize = 64 * 1024;
 /// never holds more than it, so it has to be enforced while reading. Both pipes
 /// are drained concurrently because a child that fills the one nobody reads
 /// blocks forever.
-async fn run_capturing(
+pub(crate) async fn run_captured(
     mut cmd: Command,
     name: &str,
     ctx: &mut JobCtx<'_>,
@@ -3479,7 +3594,8 @@ async fn run_capturing(
     w_id: &str,
     conn: &Connection,
     max_stdout_bytes: usize,
-) -> error::Result<String> {
+    on_overflow: Overflow,
+) -> error::Result<Captured> {
     use tokio::io::AsyncReadExt;
 
     let mut child = cmd
@@ -3514,6 +3630,7 @@ async fn run_capturing(
             let mut out_buf = vec![0u8; 16 * 1024];
             let mut err_buf = vec![0u8; 16 * 1024];
             let (mut out_open, mut err_open) = (true, true);
+            let mut truncated = false;
             while out_open || err_open {
                 tokio::select! {
                     r = stdout_pipe.read(&mut out_buf[..]), if out_open => match r {
@@ -3521,14 +3638,19 @@ async fn run_capturing(
                         Ok(n) => {
                             if stdout.len() + n > max_stdout_bytes {
                                 // Killed here rather than left to `kill_on_drop`
-                                // so the child is gone before the error unwinds,
-                                // not merely once this future is dropped.
+                                // so the child is gone before this returns, not
+                                // merely once the future is dropped.
                                 let _ = child.kill().await;
-                                return Err(Error::ExecutionErr(format!(
-                                    "{name} produced more than {} MB of output. Narrow the \
-                                     selection, or query the relation from a SQL script.",
-                                    max_stdout_bytes / 1024 / 1024
-                                )));
+                                if on_overflow == Overflow::Fail {
+                                    return Err(Error::ExecutionErr(format!(
+                                        "{name} produced more than {} MB of output. Narrow the \
+                                         selection, or query the relation from a SQL script.",
+                                        max_stdout_bytes / 1024 / 1024
+                                    )));
+                                }
+                                truncated = true;
+                                out_open = false;
+                                continue;
                             }
                             stdout.extend_from_slice(&out_buf[..n]);
                         }
@@ -3551,7 +3673,7 @@ async fn run_capturing(
                 .wait()
                 .await
                 .map_err(|e| Error::internal_err(format!("{name} failed: {e}")))?;
-            Ok((status, stdout, stderr))
+            Ok((status, stdout, stderr, truncated))
         },
         ctx.worker_name,
         w_id,
@@ -3561,14 +3683,46 @@ async fn run_capturing(
         })),
     )
     .await?;
-    let (status, stdout, stderr) = out;
-    if !status.success() {
+    let (status, stdout, stderr, truncated) = out;
+    Ok(Captured {
+        stdout: String::from_utf8_lossy(&stdout).to_string(),
+        stderr: String::from_utf8_lossy(&stderr).to_string(),
+        // A killed child reports failure; under `Truncate` that is the ceiling's
+        // doing, not the project's, and the caller reads `truncated` to tell.
+        success: status.success(),
+        truncated,
+    })
+}
+
+/// `run_captured`, with a non-zero exit folded into the error — what a caller
+/// that needs the command to have WORKED wants.
+pub(crate) async fn run_capturing(
+    cmd: Command,
+    name: &str,
+    ctx: &mut JobCtx<'_>,
+    job_id: &Uuid,
+    w_id: &str,
+    conn: &Connection,
+    max_stdout_bytes: usize,
+) -> error::Result<Captured> {
+    let captured = run_captured(
+        cmd,
+        name,
+        ctx,
+        job_id,
+        w_id,
+        conn,
+        max_stdout_bytes,
+        Overflow::Fail,
+    )
+    .await?;
+    if !captured.success {
         return Err(Error::ExecutionErr(format!(
             "{name} failed: {}",
-            String::from_utf8_lossy(&stderr)
+            captured.stderr
         )));
     }
-    Ok(String::from_utf8_lossy(&stdout).to_string())
+    Ok(captured)
 }
 
 /// Run a preparation command through the same child handler the build uses, so
@@ -4518,7 +4672,11 @@ fn has_retryable_node(run_results: &str) -> bool {
 }
 
 /// Append `--vars` if the descriptor (or the run) declares any.
-fn add_vars(cmd: &mut Command, descriptor: &DbtDescriptor, inv: &Invocation) -> error::Result<()> {
+pub(crate) fn add_vars(
+    cmd: &mut Command,
+    descriptor: &DbtDescriptor,
+    inv: &Invocation,
+) -> error::Result<()> {
     let vars = resolved_vars(descriptor, &inv.args, inv.strict)?;
     if !vars.is_empty() {
         cmd.args(["--vars", &serde_json::to_string(&vars).unwrap_or_default()]);

backend/windmill-worker/src/lib.rs

@@ -33,6 +33,7 @@ pub mod common;
 mod config;
 mod csharp_executor;
 
+mod dbt_column_index;
 mod dbt_engine;
 mod dbt_executor;
 mod dbt_profiles;

docs/dbt-runtime.md

@@ -36,7 +36,7 @@ the dominant way dbt is orchestrated today.
 | 11 | Asset kind | `dbt://<warehouse>/<schema>/<name>` — keyed on the relation, not on dbt's node id. See below |
 | 12 | Graph refresh | Deploy-time, re-ingested per run only when the descriptor is dynamic, plus an explicit `parse` of the editor's buffer. See below |
 | 13 | Manifest storage | Sidecar table for nodes/edges. Full manifest **not** stored — see below |
-| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** is not in the manifest — see below |
+| 14 | Metadata depth | Tests, strategy, tags, freshness, column descriptions. Column **lineage** and real column schemas come from the engine's parquet index, opt-in per project — see below |
 | 15 | Node rendering | Asset nodes per model plus one runnable node for the script |
 | 16 | Progress | Live, from the JSON event stream |
 | 17 | Test failures | Honor dbt's own `severity` |
@@ -857,6 +857,8 @@ profile:
 select: ["tag:nightly+"]
 exclude: []
 test_behavior: build              # build | after_all | none
+column_lineage: false             # opt in to the static-analysis pass that
+                                  # produces column-level lineage (decision 14)
 vars:                             # typed: numbers/bools/lists keep their type,
   run_date: "{{ run_date }}"      # and string leaves take job arguments
   strict: false
@@ -1151,11 +1153,118 @@ would be an unread copy of data that is already reproducible by redeploying (or,
 for a dynamic descriptor, by the next run). Worth adding the day something needs the
 parts the sidecar drops — compiled SQL, macro definitions — and not before.
 
-**Decision 14 — column lineage is not available.** The decision assumed
-`manifest.json` carries column-to-column edges; it does not, in either core
-engine. What it does carry is declared column *descriptions*, which are
-ingested. Real column lineage would need Fusion (which does static analysis) or
-a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt.
+**Decision 14 — column lineage comes from the parquet index, not the manifest.**
+`manifest.json` carries no column-to-column edges, in any engine, and its
+`columns` are the ones an author declared in `schema.yml`. Both halves exist in a
+different artifact: `dbt compile --static-analysis strict --write-index` writes
+`target/index/`, and two of its tables are `dbt.column_lineage.parquet`
+(`from_node_unique_id`, `from_column_name`, `to_node_unique_id`,
+`to_column_name`, `lineage_kind`) and `dbt.node_columns.parquet` (every column of
+every node, with its declared type, its inferred type and its description).
+
+Three measured properties decide the shape of the ingest.
+
+**Strict analysis is a stricter dialect.** `select no_such_column from
+ref(...)` is `UnresolvedIdentifier (dbt0227)` and exit 1 under `strict`, and
+compiles fine under `baseline` (the default). So this is a separate `dbt compile`
+with its own `--target-path`, never a flag on the build, and it is opt-in per
+project: `column_lineage: true` in the descriptor. Off, nothing changes. On, a
+project that cannot be analyzed keeps exactly the graph it had.
+
+The pass is best-effort about everything that is ITS: a wrong engine, a rejected
+analysis, a missing or unreadable artifact, an over-long output and outrunning its
+own time budget all degrade to partial lineage or none, plus a line in the job
+log saying which. It is not best-effort about the JOB: a cancellation or the job's
+own deadline fail it, because swallowing those would let a run that blew its
+timeout inside an optional annotation publish a graph and report success. That
+split is why the two halves have separate error contracts — the compile owns the
+job's semantics and may `Err`; reading the artifact owns none, and cannot. The
+budget is half the job's remaining wall clock, spent on the compile alone, so the
+build that follows cannot be starved by it.
+
+**A failed pass still writes the index**, holding every edge of the models that
+did analyze, so the artifact is read whatever the exit status and partial lineage
+is a normal outcome. An unreachable *source* is milder still: `RemoteError
+(dbt1014)` downgrades that model to `static_analysis: off` and the compile
+succeeds. (Strict analysis queries the warehouse catalog for source schemas; a
+`ref()`ed model is inferred statically and needs no built table.)
+
+**The flag is not the capability.** `dbt-core` 2.0.0-alpha.5 — the version
+`DBT_CORE_2X_VERSION` pins — accepts `--write-index` and `--write-lineage`, and
+its own `views.sql` declares views over both tables, but it writes neither
+parquet; only Fusion does today. The ADAPTER decides too: an experimental one
+(postgres under `DBT_ALLOW_EXPERIMENTAL_ADAPTERS`) turns static analysis off and
+says so only in a warning on an otherwise successful compile. The gate is
+therefore "the engine has the flag" (everything but 1.x, whose Python CLI has no
+such option) plus "the file appeared", so a later release picking the feature up
+needs no change here — and the job log carries the engine's own stderr whenever
+no index appears, since without it "no column lineage" has no explanation.
+
+`lineage_kind` is stored as TEXT, not an enum. Three values exist — `copy`
+(passthrough), `mod` (transformed) and `scan` (the column was read to produce the
+ROW rather than the value: a join key, a `where` predicate, a `group by`) — and
+the engine's own reader maps those three and passes anything else through, so the
+set is the engine's to extend. All three are STORED; only `copy` and `mod` are
+served, because a `scan` edge reaches every output column of its model — it would
+render as a complete bipartite graph, and it is most of what a project's index
+holds. Keeping it in the table is what lets a later "show indirect" view ask for
+it without every project being redeployed.
+
+Storage mirrors `dbt_edge` exactly: `dbt_column_edge`, keyed by (path, version,
+job) with the same composite foreign key to `script`, so a version's column
+lineage dies with the version and a run's snapshot with the sweep.
+
+**A table of its own, not `dbt_edge.column_lineage` JSONB.** Hanging the links on
+the `ref()` edge they sit beneath would inherit its clone, prune, clear and
+cascade paths for free, and it does not work: a model reading `{{ this }}` gets
+column lineage from itself to itself, and `parent_map` has no self-loop, because
+a model does not `ref()` itself. Those pairs have no `dbt_edge` row to attach to.
+The loss is not hypothetical — an incremental that selects from `{{ this }}`
+(`coalesce(p.dbl, s.dbl)`, `p.up as prev_up`) yields `up → prev_up` with kind
+`copy`, a drawn edge meaning "this column carries the previous run's value".
+Inventing self-loop `dbt_edge` rows to hold it is not an option either: that
+table is `ref()` lineage. The typed column list lands in
+`dbt_node.column_schema`, beside `columns` rather than merged into it —
+`columns` stays what the author *declared*.
+
+**Served from `assets/column_lineage`, keyed to one relation, not as a field on
+the asset graph.** The graph is folder-wide and a run page polls it, while a
+column trace is drawn for one selected node; carried on the graph the edges would
+need a cap, and a cap has to be applied after every filter that can drop a row —
+scope, project visibility, the asset set actually rendered. That ordering is what
+the separate endpoint removes rather than gets right: here the filters *are* the
+answer. The caller's `scripts:read` scope and the project's visibility are decided
+once in SQL, for the script that owns the relation; the size is bounded at ingest
+(`MAX_COLUMN_EDGES` per version, of which only the direct kinds are served); and
+what comes back is the **connected component** the relation's columns sit in,
+which is exactly what the canvas lays out. Neither the relation's own edges (a
+trace walks transitively, so that stops one hop out) nor the whole project's
+(model families the selection cannot reach). The component is walked in Rust over
+the rows the gated query returns, not by a recursive CTE: a CTE has no index to
+walk, so the recursive term rescans the whole edge set once per level — measured
+at 1.24s against 59ms for the query alone on a 3000-model project. Pinning to a
+run's snapshot or to the editor's parse of its own buffer costs the job-read gate,
+so that form is `jobs/dbt_column_lineage/{id}`, exactly as `jobs/dbt_graph/{id}`
+is to `assets/graph`.
+
+The two halves of a column trace are fetched separately and merged in the
+browser: the producer half — what a DuckDB script's `// column` annotations and
+inferred SQL lineage say — rides on the asset graph, and dbt's rides on this
+endpoint. They meet at shared node ids, since `// column total <-
+dbt://wh/analytics/orders.amount` mints the same `(dbt, path, column)` node dbt's
+own lineage does, so a trace crosses the boundary in both directions rather than
+ending at it.
+
+Both the lineage and `column_schema` are gated on being able to read the
+producing project, like the model's SQL: a column-level view is the shape of what
+the author wrote, one level finer than the `ref()` graph, which is ungated only
+because it draws relations the caller already sees. **That gate is separate from
+the pin**, and the pinned read is where the two are easiest to conflate: a run
+resolves WHICH version answers, and never whether the caller may read it. A
+share-link viewer entitled to a dbt run gets its relations and `ref()` edges and
+an empty lineage, the same split `dbt_graph` already makes by redacting
+`raw_code`. The one exemption is a version-less row — an editor buffer, which has
+no `script` row to ask and is reachable only through the parse job that wrote it.
 
 ## Concept mapping
 
@@ -1167,7 +1276,9 @@ a SQL-AST pass of our own, so `columnLineageGraph.ts` is not wired up for dbt.
 | `materialized: incremental` | `append` or `merge` (by `unique_key`) | same |
 | `{% snapshot %}` | `scd2` | same, incl. `<dim>_current` handling |
 | `unique`/`not_null`/`accepted_values`/`relationships` | `data_tests` | exact 1:1 with the four `// data_test` kinds |
-| declared column metadata | `columns` on the asset node | descriptions only; see the note below |
+| declared column metadata | `columns` on the asset node | descriptions only, from the manifest |
+| analyzed column schema | `column_schema` on the asset node | `dbt.node_columns.parquet`, opt-in |
+| column-to-column lineage | column-lineage trace | `dbt.column_lineage.parquet`, opt-in |
 | model `tags` | node badge | `tag` |
 | source freshness | `freshness` | `last_success_at` chip |
 | `run_results.json` | materialization records | `record_materialization` |

frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte

@@ -156,6 +156,9 @@
 		// resolved graph). Drives the transitive column-lineage trace shown for a
 		// selected materialized asset.
 		selectionColumnGraph?: ColumnLineageGraph
+		/** That graph still being fetched — a dbt relation's lineage is a request
+		 *  of its own, so it arrives after the selection does. */
+		selectionColumnLoading?: boolean
 		/** dbt provenance of the selected relation, when a dbt project
 		 *  materializes it — carries the model's own SQL. */
 		selectionDbt?: DbtAssetProvenance
@@ -289,6 +292,7 @@
 		onScriptRemoved,
 		selectionProducers = [],
 		selectionColumnGraph,
+		selectionColumnLoading = false,
 		selectionDbt,
 		schemaCanEvolve = true,
 		selectionForkMaterialization = undefined,
@@ -446,6 +450,15 @@
 		return scripts.length === 1 ? `${scripts[0].path}__dbt/${file}` : file
 	})
 
+	// The selected relation's own column nodes, which is what decides whether
+	// there is a trace to draw at all: a producer that declares no column lineage
+	// — or a dbt project that never asked for the analysis pass — has none.
+	let selectionColumnNodes = $derived(
+		selection?.kind === 'asset' && selectionColumnGraph
+			? assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path)
+			: []
+	)
+
 	// Bound from ScriptEditor — populated by inferAssets on every code
 	// change. Forwarded to the page so the canvas can re-derive write
 	// edges as the user edits the body (e.g. renaming a CREATE TABLE
@@ -1221,7 +1234,7 @@
 												</span>
 											</div>
 										{/if}
-										{#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0}
+										{#if selectionColumnGraph && selectionColumnNodes.length > 0}
 											<div class="border-b shrink-0">
 												<ColumnLineageTrace
 													graph={selectionColumnGraph}
@@ -1236,7 +1249,7 @@
 										</div>
 									</div>
 								{/key}
-							{:else if selectionDbt?.raw_code}
+							{:else if selectionDbt && (selectionDbt.raw_code || selectionColumnNodes.length > 0 || selectionColumnLoading)}
 								<!-- The transform behind the node. Read-only on purpose: dbt
 								     development is a local loop (`dbt run --select`, `dbt test`
 								     against a dev target), and a browser textarea over one file
@@ -1249,11 +1262,36 @@
 										<DbtIcon width={11} height={11} />
 										<span class="font-mono truncate">{dbtBundlePath ?? selectionDbt.unique_id}</span
 										>
-										<span class="ml-auto shrink-0 opacity-70">read-only · edit locally</span>
-									</div>
-									<div class="flex-1 min-h-0 overflow-auto">
-										<HighlightCode language="sql" code={selectionDbt.raw_code} />
+										{#if selectionDbt.raw_code}
+											<span class="ml-auto shrink-0 opacity-70">read-only · edit locally</span>
+										{/if}
 									</div>
+									<!-- Above the SQL rather than beside it: the columns are what
+									     the SQL below produces, so reading them in that order is
+									     the model's own shape. Same trace component the ducklake
+									     assets use — the graph is one graph across both. -->
+									{#if selectionColumnLoading && selectionColumnNodes.length === 0}
+										<div
+											class="border-b shrink-0 flex items-center gap-2 px-3 py-1.5 text-2xs text-secondary"
+										>
+											<Loader2 size={12} class="animate-spin" />
+											Loading column lineage
+										</div>
+									{:else if selectionColumnGraph && selectionColumnNodes.length > 0}
+										<div class="border-b shrink-0 overflow-auto max-h-64">
+											<ColumnLineageTrace
+												graph={selectionColumnGraph}
+												assetKind={selection.asset_kind}
+												assetPath={selection.path}
+												targetLabel={selectionDbt.unique_id}
+											/>
+										</div>
+									{/if}
+									{#if selectionDbt.raw_code}
+										<div class="flex-1 min-h-0 overflow-auto">
+											<HighlightCode language="sql" code={selectionDbt.raw_code} />
+										</div>
+									{/if}
 								</div>
 							{:else}
 								<div class="p-3 text-xs text-secondary">

frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte

@@ -76,6 +76,7 @@
 		localScriptsVersion,
 		selectionProducers = [],
 		selectionColumnGraph,
+		selectionColumnLoading = false,
 		selectionDbt,
 		schemaCanEvolve = true,
 		selectionForkMaterialization = undefined,
@@ -180,8 +181,11 @@
 		 * the selected node's source on live-reload. */
 		localScriptsVersion?: unknown
 		selectionProducers?: Array<{ kind: 'script' | 'flow'; path: string; unsaved?: boolean }>
-		/** Transitive column-lineage trace for a selected ducklake asset (route page). */
+		/** Transitive column-lineage trace for the selected asset (route page). */
 		selectionColumnGraph?: ColumnLineageGraph
+		/** That trace still being fetched — a dbt relation's is a request of its
+		 *  own, so it arrives after the selection does. */
+		selectionColumnLoading?: boolean
 		/** dbt provenance of the selected relation — carries its SQL. */
 		selectionDbt?: DbtAssetProvenance
 		schemaCanEvolve?: boolean
@@ -514,6 +518,7 @@
 						selection={activeDraft ? undefined : editor.selection}
 						selectionProducers={activeDraft ? [] : selectionProducers}
 						{selectionColumnGraph}
+						{selectionColumnLoading}
 						{selectionDbt}
 						{schemaCanEvolve}
 						{selectionForkMaterialization}

frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.test.ts

@@ -2,7 +2,10 @@ import { describe, expect, it } from 'vitest'
 import type { AssetGraphResponse } from './types'
 import {
 	buildColumnGraph,
+	buildDbtColumnGraph,
 	colNodeId,
+	mergeColumnGraphs,
+	type ColumnLineageGraph,
 	traceColumn,
 	connectedComponent,
 	assetColumnNodes,
@@ -120,6 +123,79 @@ describe('buildColumnGraph', () => {
 	})
 })
 
+describe('buildDbtColumnGraph', () => {
+	it('takes the direct kinds and drops any other', () => {
+		// `scan` means the column was read to produce the ROW — a join key, a
+		// predicate, a `group by` — so it reaches every output column of its model
+		// and is not what a column trace means. The server filters it out; this
+		// filters again, because the kind set is the engine's and an unknown one
+		// must not become an edge the trace calls data flow.
+		const g = buildDbtColumnGraph([
+			{
+				from_asset_path: 'main/s/stg',
+				from_column: 'raw_name',
+				to_asset_path: 'main/s/mart',
+				to_column: 'clean_name',
+				kind: 'mod'
+			},
+			{
+				from_asset_path: 'main/s/stg',
+				from_column: 'id',
+				to_asset_path: 'main/s/mart',
+				to_column: 'id',
+				kind: 'copy'
+			},
+			{
+				from_asset_path: 'main/s/stg',
+				from_column: 'id',
+				to_asset_path: 'main/s/mart',
+				to_column: 'clean_name',
+				kind: 'scan'
+			}
+		])
+		expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'clean_name'))).toEqual(
+			new Set([colNodeId('dbt', 'main/s/stg', 'raw_name')])
+		)
+		expect(g.up.get(colNodeId('dbt', 'main/s/mart', 'id'))).toEqual(
+			new Set([colNodeId('dbt', 'main/s/stg', 'id')])
+		)
+	})
+})
+
+describe('mergeColumnGraphs', () => {
+	it('chains a dbt column into what a producer derives from it', () => {
+		// The two halves arrive separately — the producer's from the asset graph,
+		// dbt's from its own request — and meet at the dbt node a `// column`
+		// annotation names. A trace has to cross that, or a dbt selection stops
+		// before the script consuming it.
+		const dbt = buildDbtColumnGraph([
+			{
+				from_asset_path: 'main/s/stg',
+				from_column: 'raw',
+				to_asset_path: 'main/s/mart',
+				to_column: 'clean',
+				kind: 'copy'
+			}
+		])
+		const producer: ColumnLineageGraph = {
+			nodes: new Map(),
+			up: new Map(),
+			down: new Map()
+		}
+		const src = colNodeId('dbt', 'main/s/mart', 'clean')
+		const out = colNodeId('ducklake', 'wh/report', 'total')
+		producer.nodes.set(src, { kind: 'dbt', path: 'main/s/mart', column: 'clean' })
+		producer.nodes.set(out, { kind: 'ducklake', path: 'wh/report', column: 'total' })
+		producer.up.set(out, new Set([src]))
+		producer.down.set(src, new Set([out]))
+
+		const merged = mergeColumnGraphs(dbt, producer)
+		expect(traceColumn(colNodeId('dbt', 'main/s/stg', 'raw'), merged)).toEqual(
+			new Set([colNodeId('dbt', 'main/s/stg', 'raw'), src, out])
+		)
+	})
+})
+
 describe('traceColumn', () => {
 	it('returns the full upstream + downstream impact set of a source column', () => {
 		const g = buildColumnGraph(chainGraph())

frontend/src/lib/components/assets/AssetGraph/columnLineageGraph.ts

@@ -1,6 +1,11 @@
-import type { AssetKind } from '$lib/gen'
+import type { AssetKind, DbtColumnLineage } from '$lib/gen'
 import type { AssetGraphResponse } from './types'
 
+// One column-to-column edge of a dbt project's static analysis, as the API
+// serves it. Taken from the generated client rather than restated: unlike the
+// asset graph, this response is fetched through it.
+export type DbtColumnEdge = DbtColumnLineage['edges'][number]
+
 // A node in the column-level lineage graph: one column of one asset.
 export type ColumnNode = { kind: AssetKind; path: string; column: string }
 export type ColumnNodeId = string
@@ -23,6 +28,15 @@ export type ColumnLineageGraph = {
 	down: Map<ColumnNodeId, Set<ColumnNodeId>>
 }
 
+// Direct value flow, as dbt's static analysis labels it: `copy` passes a column
+// through, `mod` transforms it. The API serves only those two — the third kind,
+// `scan`, means the column was read to produce the ROW rather than the value (a
+// join key, a `where` predicate, a `group by`), so it reaches EVERY output
+// column of the model and would draw the diagram as a complete bipartite graph.
+// Filtered here as well so a kind the engine invents cannot silently become an
+// edge the trace claims is data flow.
+const DIRECT_DBT_LINEAGE = new Set(['copy', 'mod'])
+
 // Build the column graph from a resolved asset graph. A producer's
 // `column_lineage` describes the columns of the asset it materializes; that
 // output asset is the ducklake target it writes (v1 materialize target), found
@@ -89,6 +103,59 @@ export function buildColumnGraph(graph: AssetGraphResponse): ColumnLineageGraph
 	return { nodes, up, down }
 }
 
+// The same graph, from one dbt relation's column lineage. dbt arrives already
+// resolved to two relations rather than anchored to a producer, and the API
+// serves only the direct kinds, so this is a straight edge list.
+export function buildDbtColumnGraph(edges: DbtColumnEdge[]): ColumnLineageGraph {
+	const nodes = new Map<ColumnNodeId, ColumnNode>()
+	const up = new Map<ColumnNodeId, Set<ColumnNodeId>>()
+	const down = new Map<ColumnNodeId, Set<ColumnNodeId>>()
+	const addNode = (n: ColumnNode): ColumnNodeId => {
+		const id = colNodeId(n.kind, n.path, n.column)
+		if (!nodes.has(id)) nodes.set(id, n)
+		return id
+	}
+	for (const e of edges) {
+		// Belt and braces: the API filters to `copy`/`mod`, and a kind an engine
+		// invents must not silently become an edge the trace calls data flow.
+		if (!DIRECT_DBT_LINEAGE.has(e.kind)) continue
+		const src = addNode({ kind: 'dbt', path: e.from_asset_path, column: e.from_column })
+		const out = addNode({ kind: 'dbt', path: e.to_asset_path, column: e.to_column })
+		if (src === out) continue
+		;(up.get(out) ?? up.set(out, new Set()).get(out)!).add(src)
+		;(down.get(src) ?? down.set(src, new Set()).get(src)!).add(out)
+	}
+	return { nodes, up, down }
+}
+
+// One graph out of several, so a trace crosses the boundary between them.
+//
+// The two halves reach each other through shared node ids: a producer's
+// `// column out <- dbt://wh/schema/model.col` puts a `('dbt', path, column)`
+// node in the producer graph under the same `colNodeId` the dbt lineage mints
+// for it, so the union chains a dbt model's columns into the script that
+// consumes them and on into what that script writes. Kept separate up to here
+// because they are fetched separately — the producer half rides on the asset
+// graph, the dbt half is asked for per selection.
+export function mergeColumnGraphs(...graphs: ColumnLineageGraph[]): ColumnLineageGraph {
+	const nodes = new Map<ColumnNodeId, ColumnNode>()
+	const up = new Map<ColumnNodeId, Set<ColumnNodeId>>()
+	const down = new Map<ColumnNodeId, Set<ColumnNodeId>>()
+	for (const g of graphs) {
+		for (const [id, n] of g.nodes) if (!nodes.has(id)) nodes.set(id, n)
+		for (const [dir, into] of [
+			[g.up, up],
+			[g.down, down]
+		] as const) {
+			for (const [id, adj] of dir) {
+				const target = into.get(id) ?? into.set(id, new Set()).get(id)!
+				for (const m of adj) target.add(m)
+			}
+		}
+	}
+	return { nodes, up, down }
+}
+
 // Every node reachable from `start` by following `adj` (transitive closure,
 // excluding `start` itself). Iterative to avoid deep-recursion limits.
 function reach(start: ColumnNodeId, adj: Map<ColumnNodeId, Set<ColumnNodeId>>): Set<ColumnNodeId> {

frontend/src/lib/components/assets/AssetGraph/dbtColumnLineage.svelte.ts

@@ -0,0 +1,132 @@
+import { untrack } from 'svelte'
+import { AssetService, JobService } from '$lib/gen'
+import {
+	buildDbtColumnGraph,
+	mergeColumnGraphs,
+	type ColumnLineageGraph
+} from './columnLineageGraph'
+
+export const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = {
+	nodes: new Map(),
+	up: new Map(),
+	down: new Map()
+}
+
+/** Which stored dbt graph a view is drawing. A job — a run's snapshot, or the
+ *  editor's parse of its own buffer — is asked through the job route, the only
+ *  way to reach a graph that names no deployed version; otherwise the deployed
+ *  version by hash, or the current one when there is no hash. */
+export type DbtGraphPin = { jobId?: string; scriptHash?: string | number }
+
+/** What a dbt relation's column lineage is doing right now. `loading` is
+ *  separate because a project still being fetched and one that never asked for
+ *  the analysis pass are the same empty graph otherwise. */
+export type DbtColumnLineageState = {
+	readonly graph: ColumnLineageGraph
+	readonly loading: boolean
+}
+
+function pinKey(workspace: string, pin: DbtGraphPin | undefined): string {
+	return `${workspace}|${pin?.jobId ?? ''}|${pin?.scriptHash ?? ''}`
+}
+
+function fetchComponent(
+	workspace: string,
+	assetPath: string,
+	pin: DbtGraphPin | undefined
+): Promise<ColumnLineageGraph> {
+	const req = pin?.jobId
+		? JobService.getDbtRunColumnLineage({ workspace, id: pin.jobId, assetPath })
+		: AssetService.getDbtColumnLineage({
+				workspace,
+				assetPath,
+				dbtScriptHash: pin?.scriptHash != undefined ? String(pin.scriptHash) : undefined
+			})
+	return req.then(
+		(r) => buildDbtColumnGraph(r?.edges ?? []),
+		// Lineage annotates a graph that renders without it, so a failed fetch
+		// leaves that branch unexpanded rather than putting an error over the
+		// model — and one failed boundary does not lose the others.
+		() => EMPTY_COLUMN_GRAPH
+	)
+}
+
+/** Follow the selection, fetching the dbt column lineage it reaches.
+ *
+ *  Per asset rather than off the graph response: the graph is folder-wide and a
+ *  run page polls it, while this is drawn for one selection. It also means the
+ *  request is never made for a project that did not opt into the analysis pass
+ *  — the pane simply never shows the section.
+ */
+export function useDbtColumnLineage(args: {
+	workspace: () => string | undefined
+	/** The dbt relations to expand. The selection itself when it is one; for a
+	 *  selection of another kind, every dbt relation its own lineage reaches —
+	 *  a ducklake table can be derived from several, and expanding only the
+	 *  first would leave the rest as leaves. */
+	assetPaths: () => string[]
+	/** The graph on screen, so the lineage describes the same project. */
+	pin?: () => DbtGraphPin | undefined
+}): DbtColumnLineageState {
+	let graph = $state<ColumnLineageGraph>(EMPTY_COLUMN_GRAPH)
+	let loading = $state(false)
+
+	// Which pin the graph in hand was fetched against, and which relations were
+	// actually ASKED about under it.
+	let heldPin: string | undefined = undefined
+	let asked = new Set<string>()
+
+	$effect(() => {
+		const workspace = args.workspace()
+		const paths = args.assetPaths()
+		const pin = args.pin?.()
+		if (!workspace || paths.length === 0) {
+			graph = EMPTY_COLUMN_GRAPH
+			heldPin = undefined
+			asked = new Set()
+			loading = false
+			return
+		}
+		const key = pinKey(workspace, pin)
+		// `untrack`: this effect writes `graph`, so reading it as a dependency
+		// would make it retrigger itself forever.
+		const fresh = heldPin !== key
+		const base = untrack(() => (fresh ? EMPTY_COLUMN_GRAPH : graph))
+		if (fresh) asked = new Set()
+		// Only a relation this pin has ASKED about is skipped, not every relation
+		// present in what came back. A relation two projects describe has an owner
+		// row in each, and a component fetched for one of them carries that
+		// relation as an endpoint without the other project's half — so treating
+		// "appears in the graph" as "resolved" would hide exactly the cross-project
+		// edges the server's relation-keyed walk exists to merge.
+		const missing = paths.filter((p) => !asked.has(p))
+		if (missing.length === 0) {
+			graph = base
+			loading = false
+			return
+		}
+		// A selection changes faster than a request completes, so an answer is
+		// applied only while it is still the one being asked for.
+		let current = true
+		loading = true
+		Promise.all(missing.map((p) => fetchComponent(workspace, p, pin))).then((parts) => {
+			if (!current) return
+			graph = mergeColumnGraphs(base, ...parts)
+			heldPin = key
+			for (const p of missing) asked.add(p)
+			loading = false
+		})
+		return () => {
+			current = false
+		}
+	})
+
+	return {
+		get graph() {
+			return graph
+		},
+		get loading() {
+			return loading
+		}
+	}
+}

frontend/src/lib/components/assets/AssetGraph/types.ts

@@ -36,9 +36,14 @@ export interface DbtAssetProvenance {
 	tags?: string[]
 	description?: string
 	data_tests?: DbtDataTest[]
-	/** Declared column metadata (name -> description). NOT column lineage:
-	 *  `manifest.json` carries none (docs/dbt-runtime.md, decision 14). */
+	/** Declared column metadata (name -> description): what `manifest.json`
+	 *  carries, which is only the columns an author wrote down. */
 	columns?: Record<string, string>
+	/** Every column of the relation, typed and in the order the model produces
+	 *  them, from the engine's static analysis. Present only for a project that
+	 *  opted into it (`column_lineage: true`); `manifest.json` has no such
+	 *  thing. Lockstep with Rust `DbtAssetProvenance.column_schema`. */
+	column_schema?: { name: string; type?: string }[]
 	/** A source's declared freshness policy. */
 	freshness?: unknown
 	/** The model's SQL as written — the transform behind the node. Read-only:

frontend/src/lib/components/dbt/DbtEditor.svelte

@@ -31,6 +31,15 @@
 		AssetGraphNodeData,
 		DbtAssetProvenance
 	} from '$lib/components/assets/AssetGraph/types'
+	import {
+		EMPTY_COLUMN_GRAPH,
+		useDbtColumnLineage,
+		type DbtGraphPin
+	} from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte'
+	import {
+		mergeColumnGraphs,
+		type ColumnLineageGraph
+	} from '$lib/components/assets/AssetGraph/columnLineageGraph'
 	import {
 		DBT_DESCRIPTOR,
 		DBT_MODULE_EXTENSIONS,
@@ -207,6 +216,28 @@
 	let graphSelection = $state<AssetGraphNodeData | undefined>(undefined)
 	let selectedAsset = $derived(graphSelection?.kind === 'asset' ? graphSelection : undefined)
 	let selectedDbt = $state<DbtAssetProvenance | undefined>(undefined)
+	// Which graph the selection came from, so the lineage fetched below is the
+	// selected node's own project rather than whatever is deployed.
+	let selectionPin = $state<DbtGraphPin | undefined>(undefined)
+	// The selected model's column lineage, fetched on selection. Its own request
+	// rather than a field on the graph: only a project that opted into the
+	// analysis pass has any, and it is drawn for one model at a time.
+	const columnLineage = useDbtColumnLineage({
+		workspace: () => opWs,
+		assetPaths: () => {
+			const path = selectedDbt ? selectedAsset?.path : undefined
+			return path ? [path] : []
+		},
+		pin: () => selectionPin
+	})
+	// What the scripts around this project declare about its columns, off the
+	// same graph response the canvas drew. Merged rather than chosen between: a
+	// model's column and the ducklake column a script derives from it are one
+	// chain, and the trace has to cross that boundary.
+	let selectionProducerColumns = $state<ColumnLineageGraph>(EMPTY_COLUMN_GRAPH)
+	let selectionColumnGraph = $derived(
+		mergeColumnGraphs(columnLineage.graph, selectionProducerColumns)
+	)
 	// Set when the selected node came from a buffer parse: the project that parse
 	// ran on, which is the one its rows must come from. Undefined for a node off
 	// the deployed graph, which previews by version instead. Either way the rows
@@ -514,10 +545,12 @@
 						testRunning={testIsLoading}
 						testResult={testJob?.result}
 						selection={graphSelection}
-						onSelect={(sel, dbt, buffer) => {
+						onSelect={(sel, dbt, buffer, pin, producerColumns) => {
 							graphSelection = sel
 							selectedDbt = dbt
 							selectedBuffer = buffer
+							selectionPin = pin
+							selectionProducerColumns = producerColumns
 						}}
 					/>
 				</Pane>
@@ -539,6 +572,8 @@
 							{args}
 							fileInBundle={!!selectedDbt.original_file_path &&
 								!!modules?.[selectedDbt.original_file_path]}
+							columnGraph={selectionColumnGraph}
+							columnLoading={columnLineage.loading}
 							onOpenFile={open}
 							onClose={() => (graphSelection = undefined)}
 						/>

frontend/src/lib/components/dbt/DbtModelDetails.svelte

@@ -12,6 +12,11 @@
 	import { ClipboardCopy, Code2, FileCode2, Loader2, TableProperties, X } from 'lucide-svelte'
 	import { copyToClipboard } from '$lib/utils'
 	import type { DbtAssetProvenance } from '$lib/components/assets/AssetGraph/types'
+	import ColumnLineageTrace from '$lib/components/assets/AssetGraph/ColumnLineageTrace.svelte'
+	import {
+		assetColumnNodes,
+		type ColumnLineageGraph
+	} from '$lib/components/assets/AssetGraph/columnLineageGraph'
 	import { previewDbtRows, type DbtPreview, type DbtPreviewBuffer } from './previewRows'
 	import { nodeSelector } from './parseDbtRun'
 
@@ -34,6 +39,14 @@
 		args,
 		/** Whether this model's file is in the bundle being edited. */
 		fileInBundle = false,
+		/** The project's column-level lineage, when the descriptor asked for it.
+		 *  Fetched for this relation against the same graph the canvas draws, so
+		 *  the trace and the nodes above it describe one parse. */
+		columnGraph,
+		/** That fetch still in flight. Distinguished from an empty graph: a
+		 *  project without the analysis pass shows nothing at all, and a slow
+		 *  answer must not read as that. */
+		columnLoading = false,
 		onOpenFile,
 		onClose
 	}: {
@@ -45,6 +58,8 @@
 		buffer?: DbtPreviewBuffer
 		args?: Record<string, unknown>
 		fileInBundle?: boolean
+		columnGraph?: ColumnLineageGraph
+		columnLoading?: boolean
 		onOpenFile?: (path: string) => void
 		onClose?: () => void
 	} = $props()
@@ -111,7 +126,27 @@
 		return typeof v === 'object' ? JSON.stringify(v) : String(v)
 	}
 
-	let columns = $derived(Object.entries(dbt.columns ?? {}))
+	// The real columns where the analysis pass produced them — typed and in the
+	// order the model emits them — and the declared ones otherwise. The
+	// description comes from `columns` either way: that is the only place an
+	// author's prose lives, and a project documents a handful of forty.
+	let columns = $derived(
+		dbt.column_schema?.length
+			? dbt.column_schema.map((c) => ({
+					name: c.name,
+					type: c.type,
+					description: dbt.columns?.[c.name] ?? ''
+				}))
+			: Object.entries(dbt.columns ?? {}).map(([name, description]) => ({
+					name,
+					type: undefined,
+					description
+				}))
+	)
+	let columnsAreAnalyzed = $derived(!!dbt.column_schema?.length)
+	// The selected relation's own column nodes: empty for a project that never
+	// asked for the analysis pass, which is the ordinary case.
+	let columnNodes = $derived(columnGraph ? assetColumnNodes(columnGraph, 'dbt', assetPath) : [])
 	// `dbt show` SELECTs from the node's own relation and the worker intersects
 	// the selector with `resource_type:model`, so offering it on a seed, snapshot
 	// or source only ever produces a failed job.
@@ -202,15 +237,15 @@
 
 	{#if stalePlaceholders}
 		<div class="shrink-0 px-2 py-1 border-b text-2xs text-secondary bg-surface-secondary">
-			The run arguments have changed since this graph was parsed, so these rows need not
-			describe the models on screen — arguments reach schemas, aliases and which models exist
-			at all. Refresh the models to draw and preview them under the current ones.
+			The run arguments have changed since this graph was parsed, so these rows need not describe
+			the models on screen — arguments reach schemas, aliases and which models exist at all. Refresh
+			the models to draw and preview them under the current ones.
 		</div>
 	{:else if staleVars}
 		<div class="shrink-0 px-2 py-1 border-b text-2xs text-secondary bg-surface-secondary">
-			The run form's vars have changed since this graph was parsed. Rows are previewed under
-			the vars it was parsed with, so they still describe the models on screen — refresh the
-			models to draw and preview them under the current ones.
+			The run form's vars have changed since this graph was parsed. Rows are previewed under the
+			vars it was parsed with, so they still describe the models on screen — refresh the models to
+			draw and preview them under the current ones.
 		</div>
 	{/if}
 
@@ -238,20 +273,29 @@
 			<div class="px-2 py-1.5 border-b flex flex-col gap-1.5">
 				{#if columns.length > 0}
 					<div class="text-2xs">
-						<div class="text-tertiary mb-0.5">columns declared</div>
+						<div class="text-tertiary mb-0.5">
+							{columnsAreAnalyzed ? 'columns' : 'columns declared'}
+						</div>
 						<div class="flex flex-col gap-0.5">
-							{#each columns as [name, desc] (name)}
+							{#each columns as col (col.name)}
 								<div class="flex gap-2">
-									<span class="font-mono text-primary shrink-0">{name}</span>
-									<span class="text-secondary truncate">{desc}</span>
+									<span class="font-mono text-primary shrink-0">{col.name}</span>
+									{#if col.type}
+										<span class="font-mono text-tertiary shrink-0">{col.type}</span>
+									{/if}
+									<span class="text-secondary truncate">{col.description}</span>
 								</div>
 							{/each}
 						</div>
-						<!-- dbt's manifest carries no column-to-column edges, so this is a
-						     declared column SET rather than lineage. -->
-						<div class="text-tertiary mt-0.5">
-							Declared metadata — dbt reports no column-level lineage.
-						</div>
+						<!-- `manifest.json` carries declared columns only, so without the
+						     analysis pass this list is what an author wrote down rather than
+						     what the model produces. -->
+						{#if !columnsAreAnalyzed}
+							<div class="text-tertiary mt-0.5">
+								Declared metadata. Set `column_lineage: true` in the descriptor for the real column
+								schema and column-level lineage.
+							</div>
+						{/if}
 					</div>
 				{/if}
 				{#if (dbt.data_tests?.length ?? 0) > 0}
@@ -269,6 +313,22 @@
 			</div>
 		{/if}
 
+		{#if columnLoading && columnNodes.length === 0}
+			<div class="border-b flex items-center gap-2 p-2 text-2xs text-secondary">
+				<Loader2 size={12} class="animate-spin" />
+				Loading column lineage
+			</div>
+		{:else if columnGraph && columnNodes.length > 0}
+			<div class="border-b overflow-auto max-h-64">
+				<ColumnLineageTrace
+					graph={columnGraph}
+					assetKind="dbt"
+					{assetPath}
+					targetLabel={dbt.unique_id}
+				/>
+			</div>
+		{/if}
+
 		{#if showRows && preview}
 			{#if 'error' in preview}
 				<div class="p-2 text-2xs text-secondary">{preview.error}</div>

frontend/src/lib/components/dbt/DbtModelGraph.svelte

@@ -25,6 +25,14 @@
 		DbtAssetProvenance
 	} from '$lib/components/assets/AssetGraph/types'
 	import { useDbtRunStatus } from './runStatus.svelte'
+	import {
+		EMPTY_COLUMN_GRAPH,
+		type DbtGraphPin
+	} from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte'
+	import {
+		buildColumnGraph,
+		type ColumnLineageGraph
+	} from '$lib/components/assets/AssetGraph/columnLineageGraph'
 
 	let {
 		workspace,
@@ -80,7 +88,19 @@
 			 *  buffer rather than a deployed version — as submitted, not as the
 			 *  editor holds it now. Sent with the selection rather than exposed on
 			 *  its own so it can never disagree with the SQL the parent shows. */
-			buffer: DbtPreviewBuffer | undefined
+			buffer: DbtPreviewBuffer | undefined,
+			/** Which graph this node was taken from, so anything else fetched
+			 *  about it describes the same project: the editor's own parse job
+			 *  when the panel is pinned to one, else the deployed version. Sent
+			 *  with the selection for the same reason the buffer is — it must not
+			 *  be able to disagree with the node on screen. */
+			pin: DbtGraphPin,
+			/** Column lineage the CONSUMERS of this project declare — a script
+			 *  reading a model's column and writing a ducklake one. It comes off
+			 *  the same graph response, and the details pane merges it with the
+			 *  project's own so a trace crosses that boundary instead of ending
+			 *  at it. */
+			producerColumns: ColumnLineageGraph
 		) => void
 	} = $props()
 
@@ -364,6 +384,19 @@
 	// graph that actually came back.
 	let editorParsed = $derived(refreshJob != undefined && raw?.dbt_snapshot_job === refreshJob)
 
+	// Which stored graph is on screen. Anything the details pane fetches about a
+	// selected node asks for this one, so it cannot describe a node parsed from
+	// the buffer with the deployed version's answer.
+	let pin = $derived<DbtGraphPin>(
+		editorParsed && refreshJob ? { jobId: refreshJob } : { scriptHash: deployedHash }
+	)
+
+	// What the scripts around this project declare about its columns. Empty for
+	// the ordinary project nothing downstream annotates.
+	let producerColumns = $derived(
+		graph ? buildColumnGraph(graph) : EMPTY_COLUMN_GRAPH
+	)
+
 	// `untrack`, because the effect that reloads the graph clears the selection
 	// through here: reading the graph to describe a selection would subscribe that
 	// effect to the very state its own fetch writes, and it would reload forever.
@@ -374,7 +407,9 @@
 				sel?.kind === 'asset'
 					? graph?.assets.find((a) => a.kind === sel.asset_kind && a.path === sel.path)?.dbt
 					: undefined,
-				editorParsed ? parsedBuffer : undefined
+				editorParsed ? parsedBuffer : undefined,
+				pin,
+				producerColumns
 			)
 		)
 	}
@@ -405,7 +440,6 @@
 		if (deployedHash != undefined) return 'as of last deploy'
 		return 'never parsed'
 	})
-
 </script>
 
 <div class="flex flex-col h-full min-h-0">
@@ -435,8 +469,8 @@
 
 	{#if refreshPending}
 		<div class="shrink-0 px-2 py-1.5 border-b text-2xs text-secondary">
-			Still parsing. A cold worker provisions the dbt engine before it starts; a project
-			pinned to a worker tag nothing serves waits here indefinitely.
+			Still parsing. A cold worker provisions the dbt engine before it starts; a project pinned to a
+			worker tag nothing serves waits here indefinitely.
 			<a
 				class="text-blue-500 hover:underline"
 				href="{base}/run/{refreshPending}?workspace={workspace}"

frontend/src/lib/script_helpers.ts

@@ -1381,6 +1381,11 @@ full_refresh: false
 # resolved to that Windmill variable, so secrets stay out of this file.
 # env:
 #   DBT_PASSWORD: $var:u/user/my_warehouse_password
+# Column-level lineage and real column schemas, from the engine's static
+# analysis. Opt-in because it runs a separate dbt compile under
+# --static-analysis strict, which rejects SQL the default accepts; a project it
+# cannot analyze keeps the graph it has. Needs an engine that computes it.
+# column_lineage: true
 `
 // for related places search: ADD_NEW_LANG
 export const INITIAL_CODE = {

frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte

@@ -33,9 +33,15 @@
 	import MacroExplorerDrawer from '$lib/components/assets/AssetGraph/MacroExplorerDrawer.svelte'
 	import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations'
 	import {
+		assetColumnNodes,
 		buildColumnGraph,
-		type ColumnLineageGraph
+		connectedComponent,
+		mergeColumnGraphs
 	} from '$lib/components/assets/AssetGraph/columnLineageGraph'
+	import {
+		EMPTY_COLUMN_GRAPH,
+		useDbtColumnLineage
+	} from '$lib/components/assets/AssetGraph/dbtColumnLineage.svelte'
 	import { resolveGraph } from '$lib/components/assets/AssetGraph/resolveGraph'
 	import { normalizePipelineFolder } from '$lib/utils/pipelineFolder'
 	import { hideDbtRunnables } from '$lib/components/assets/AssetGraph/hideDbtRunnables'
@@ -1980,26 +1986,51 @@
 			?.dbt
 	})
 
-	// Empty graph reused when the trace isn't shown (no ducklake-asset selection,
-	// or a draft is actively edited) so the pane blanks out like the other
-	// selection overlays and `buildColumnGraph` doesn't run.
-	const EMPTY_COLUMN_GRAPH: ColumnLineageGraph = {
-		nodes: new Map(),
-		up: new Map(),
-		down: new Map()
-	}
 	// Pipeline-wide column-lineage graph, stitched across every producer's
 	// (inferred + annotated) `column_lineage` and the asset write-edges. Drives
 	// the transitive column trace in the details pane. Built from `displayGraph`
 	// — the exact graph the canvas renders — so the trace matches it: draft
-	// overlays in edit / show-drafts, deployed-only in plain View. Gated to a
-	// ducklake-asset selection so it isn't rebuilt on every editor keystroke when
-	// the trace UI isn't even shown.
-	let columnGraph = $derived(
-		pe.selection?.kind === 'asset' && pe.selection.asset_kind === 'ducklake'
+	// overlays in edit / show-drafts, deployed-only in plain View. Gated to the
+	// two asset kinds that can carry column lineage so it isn't rebuilt on every
+	// editor keystroke when the trace UI isn't even shown.
+	let producerColumnGraph = $derived(
+		pe.selection?.kind === 'asset' &&
+			(pe.selection.asset_kind === 'ducklake' || pe.selection.asset_kind === 'dbt')
 			? buildColumnGraph(displayGraph)
 			: EMPTY_COLUMN_GRAPH
 	)
+	// dbt's half comes from its own request instead: a project's static analysis
+	// is stored per relation and only exists if the descriptor asked for it, so
+	// the folder-wide graph does not carry it. A draft is never asked about —
+	// nothing has parsed it, so there is nothing to fetch.
+	//
+	// The relation to ask about is the selected one when it IS a dbt relation,
+	// and otherwise the dbt column a producer feeding this selection names as a
+	// source — the boundary node above. Asking there is what lets a ducklake
+	// selection trace back up the dbt project that fed it, rather than stopping
+	// at the annotation.
+	let dbtSeedPaths = $derived.by(() => {
+		const sel = pe.selection
+		if (pe.activeDraft || sel?.kind !== 'asset') return []
+		if (sel.asset_kind === 'dbt') return [sel.path]
+		// EVERY dbt relation this selection reaches, not the first: one output can
+		// be derived from several, and expanding one would leave the others as
+		// leaves on the canvas.
+		const seeds = assetColumnNodes(producerColumnGraph, sel.asset_kind, sel.path)
+		const paths = new Set<string>()
+		for (const id of connectedComponent(seeds, producerColumnGraph)) {
+			const node = producerColumnGraph.nodes.get(id)
+			if (node?.kind === 'dbt') paths.add(node.path)
+		}
+		return [...paths]
+	})
+	const dbtColumnLineage = useDbtColumnLineage({
+		workspace: () => $workspaceStore,
+		assetPaths: () => dbtSeedPaths
+	})
+	// One graph across both, so a trace crosses the dbt/ducklake boundary in
+	// either direction rather than stopping at it.
+	let columnGraph = $derived(mergeColumnGraphs(producerColumnGraph, dbtColumnLineage.graph))
 
 	// Producer-side facts for the editor's live schema-contract diagnostics:
 	// which assets are muted (`on_schema_change=ignore`) and which `_current`
@@ -2602,6 +2633,7 @@
 				{selectionProducers}
 				{selectionDbt}
 				selectionColumnGraph={pe.activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph}
+				selectionColumnLoading={dbtColumnLineage.loading}
 				{schemaCanEvolve}
 				{selectionForkMaterialization}
 				{schemaContractContext}