feat: ducklake materialization for data pipelines (#9689)

backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO materialized_partition\n           (workspace_id, asset_kind, asset_path, partition, status,\n            snapshot_id, row_count, job_id, materialized_at, error)\n         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n         ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n         DO UPDATE SET status = EXCLUDED.status,\n                       snapshot_id = EXCLUDED.snapshot_id,\n                       row_count = EXCLUDED.row_count,\n                       job_id = EXCLUDED.job_id,\n                       materialized_at = now(),\n                       error = EXCLUDED.error",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        {
+          "Custom": {
+            "name": "asset_kind",
+            "kind": {
+              "Enum": [
+                "s3object",
+                "resource",
+                "variable",
+                "ducklake",
+                "datatable",
+                "volume"
+              ]
+            }
+          }
+        },
+        "Varchar",
+        "Text",
+        {
+          "Custom": {
+            "name": "materialization_status",
+            "kind": {
+              "Enum": [
+                "running",
+                "materialized",
+                "failed"
+              ]
+            }
+          }
+        },
+        "Int8",
+        "Int8",
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19"
+}

backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json

@@ -0,0 +1,111 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT asset_kind AS \"asset_kind: AssetKind\", asset_path, partition,\n                  status AS \"status: MaterializationStatus\", snapshot_id,\n                  row_count, job_id, materialized_at, error\n             FROM materialized_partition\n            WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3\n            ORDER BY partition DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "asset_kind: AssetKind",
+        "type_info": {
+          "Custom": {
+            "name": "asset_kind",
+            "kind": {
+              "Enum": [
+                "s3object",
+                "resource",
+                "variable",
+                "ducklake",
+                "datatable",
+                "volume"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "asset_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "partition",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 3,
+        "name": "status: MaterializationStatus",
+        "type_info": {
+          "Custom": {
+            "name": "materialization_status",
+            "kind": {
+              "Enum": [
+                "running",
+                "materialized",
+                "failed"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 4,
+        "name": "snapshot_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 5,
+        "name": "row_count",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 6,
+        "name": "job_id",
+        "type_info": "Uuid"
+      },
+      {
+        "ordinal": 7,
+        "name": "materialized_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 8,
+        "name": "error",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "asset_kind",
+            "kind": {
+              "Enum": [
+                "s3object",
+                "resource",
+                "variable",
+                "ducklake",
+                "datatable",
+                "volume"
+              ]
+            }
+          }
+        },
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3"
+}

backend/Cargo.lock

@@ -14354,6 +14354,7 @@ dependencies = [
  "windmill-parser",
  "windmill-parser-py",
  "windmill-parser-py-asset",
+ "windmill-parser-sql",
  "windmill-parser-sql-asset",
  "windmill-parser-ts",
  "windmill-parser-ts-asset",

backend/ee-repo-ref.txt

@@ -1 +1 @@
-ba677ea142011462ad4dfe77e8375a6dd274cdef
+23b5f55a943dd4d4f72a5406398b68f22782a8b8

backend/migrations/20260619170118_add_materialized_partition.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS idx_materialized_partition_asset_status;
+DROP TABLE IF EXISTS materialized_partition;
+DROP TYPE IF EXISTS MATERIALIZATION_STATUS;

backend/migrations/20260619170118_add_materialized_partition.up.sql

@@ -0,0 +1,29 @@
+-- Per-partition materialization state for managed `// materialize` assets.
+-- One row per (asset, partition): the latest materialization of that slice.
+-- Drives: the partition-status grid (CE observability), run-stale/gap
+-- detection, and the EE backfill worklist (missing/failed partitions). The
+-- `partition` column uses '' as the sentinel for an unpartitioned (whole-table)
+-- materialization, since partition is part of the primary key and cannot be
+-- NULL.
+CREATE TYPE MATERIALIZATION_STATUS AS ENUM ('running', 'materialized', 'failed');
+
+CREATE TABLE IF NOT EXISTS materialized_partition (
+  workspace_id     VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE,
+  asset_kind       ASSET_KIND NOT NULL,
+  asset_path       VARCHAR(255) NOT NULL,
+  partition        TEXT NOT NULL DEFAULT '',
+  status           MATERIALIZATION_STATUS NOT NULL,
+  -- DuckLake snapshot id produced by the write; NULL while running / on
+  -- failure. The pin that makes downstream reads reproducible.
+  snapshot_id      BIGINT,
+  row_count        BIGINT,
+  job_id           UUID,
+  materialized_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
+  error            TEXT,
+  PRIMARY KEY (workspace_id, asset_kind, asset_path, partition)
+);
+
+-- Backfill enumeration / grid "show only gaps": filter an asset's partitions
+-- by status without scanning the whole table.
+CREATE INDEX IF NOT EXISTS idx_materialized_partition_asset_status
+  ON materialized_partition (workspace_id, asset_kind, asset_path, status);

backend/parsers/windmill-parser/src/asset_parser.rs

@@ -107,6 +107,11 @@ pub struct ParseAssetsOutput {
     // The delay is a raw duration string parsed at deploy (parser-light).
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub retry: Option<RetrySpec>,
+    // `// materialize [manual] <asset> [append] [key=<col>]` —
+    // managed-materialization target + its strategy. At most one per script.
+    // Drives the worker's write-strategy + snapshot capture.
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub materialize: Option<MaterializeSpec>,
 }
 
 #[derive(Serialize, Debug, PartialEq, Clone)]
@@ -209,6 +214,27 @@ pub struct RetrySpec {
     pub delay: Option<String>,
 }
 
+// `// materialize [manual] <asset> [append] [key=<col>]` — declares that this
+// script produces a *managed* materialization of `<asset>` (a `ducklake://`
+// table). By default the runtime generates the write DDL around the script's
+// single trailing `SELECT` and owns idempotency, partition-state and snapshot
+// capture. `manual` is the escape hatch: the script writes its own DDL and the
+// runtime only records state (track-only). The reconciliation strategy options
+// (`append`, `key=<col>`) apply to managed mode: none → DELETE-by-partition +
+// INSERT (replace); `key=<col>` → MERGE (dedup within slice); `append` →
+// INSERT-only. `append` wins if both are given (deploy-time warning).
+#[derive(Serialize, Debug, PartialEq, Clone)]
+pub struct MaterializeSpec {
+    pub target_kind: AssetKind,
+    pub target_path: String,
+    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
+    pub manual: bool,
+    #[serde(skip_serializing_if = "std::ops::Not::not", default)]
+    pub append: bool,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub unique_key: Option<String>,
+}
+
 // `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger
 // firing runs the script (current behaviour). `All` = AND: the script
 // runs only once every partition-bearing input has materialized at the
@@ -239,6 +265,7 @@ pub struct PipelineAnnotations {
     pub debounce_default: Option<String>,
     pub tag: Option<String>,
     pub retry: Option<RetrySpec>,
+    pub materialize: Option<MaterializeSpec>,
 }
 
 impl ParseAssetsOutput {
@@ -262,6 +289,7 @@ impl ParseAssetsOutput {
             debounce_default: pipeline.debounce_default,
             tag: pipeline.tag,
             retry: pipeline.retry,
+            materialize: pipeline.materialize,
         }
     }
 }
@@ -571,6 +599,15 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations {
             continue;
         }
 
+        if let Some(after_kw) = consume_keyword(rest, "materialize") {
+            if out.materialize.is_none() {
+                if let Some(spec) = parse_materialize_spec(after_kw.trim()) {
+                    out.materialize = Some(spec);
+                }
+            }
+            continue;
+        }
+
         if let Some(after_kw) = consume_keyword(rest, "on") {
             let spec_text = after_kw.trim();
             if spec_text.is_empty() {
@@ -618,6 +655,35 @@ fn parse_retry_spec(s: &str) -> Option<RetrySpec> {
     Some(RetrySpec { count, delay })
 }
 
+// Parse a `// materialize [manual] <asset> [append] [key=<col>]` right-hand
+// side. An optional leading `manual` token (whitespace-delimited) opts out of
+// managed mode (track-only). The next whitespace token is the target asset URI
+// (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the
+// remainder are strategy options — bare `append` and `key=<col>` (merge key),
+// which apply to managed mode only. A missing/empty target yields `None` (the
+// annotation is dropped, fail-safe).
+fn parse_materialize_spec(s: &str) -> Option<MaterializeSpec> {
+    let (manual, rest) = match s.strip_prefix("manual") {
+        Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => {
+            (true, after.trim_start())
+        }
+        _ => (false, s),
+    };
+    let mut it = rest.trim().splitn(2, char::is_whitespace);
+    let asset_tok = it.next()?;
+    let opts_str = it.next().unwrap_or("");
+    let (target_kind, path) = parse_asset_syntax(asset_tok.trim(), true)?;
+    if path.is_empty() {
+        return None;
+    }
+    let append = opts_str.split_whitespace().any(|t| t == "append");
+    let unique_key = parse_kv_opts(opts_str)
+        .get("key")
+        .filter(|k| !k.is_empty())
+        .cloned();
+    Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key })
+}
+
 // Parse a `// partitioned <kind> [opts]` right-hand side. Recognized kinds:
 // `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start),
 // and `dynamic key="<jsonpath>"` (plus optional format).
@@ -1044,6 +1110,67 @@ mod pipeline_annotation_tests {
         assert!(out.retry.is_none());
     }
 
+    #[test]
+    fn materialize_managed_default() {
+        let out = parse_pipeline_annotations("// materialize ducklake://analytics/orders_daily");
+        let m = out.materialize.expect("materialize");
+        assert_eq!(m.target_kind, AssetKind::Ducklake);
+        assert_eq!(m.target_path, "analytics/orders_daily");
+        // managed by default; replace strategy (no append / key)
+        assert!(!m.manual);
+        assert!(!m.append);
+        assert_eq!(m.unique_key, None);
+    }
+
+    #[test]
+    fn materialize_manual_escape_hatch() {
+        let out =
+            parse_pipeline_annotations("// materialize manual ducklake://analytics/orders_daily");
+        let m = out.materialize.expect("materialize");
+        assert!(m.manual);
+        assert_eq!(m.target_path, "analytics/orders_daily");
+    }
+
+    #[test]
+    fn materialize_merge_and_append_options() {
+        let out =
+            parse_pipeline_annotations("// materialize ducklake://a/orders_daily key=order_id");
+        let m = out.materialize.expect("materialize");
+        assert_eq!(m.unique_key.as_deref(), Some("order_id"));
+        assert!(!m.append);
+
+        let out = parse_pipeline_annotations("// materialize ducklake://a/events append");
+        let m = out.materialize.expect("materialize");
+        assert!(m.append);
+        assert_eq!(m.unique_key, None);
+    }
+
+    #[test]
+    fn materialize_default_syntax_shorthand() {
+        let out = parse_pipeline_annotations("// materialize ducklake");
+        let m = out.materialize.expect("materialize");
+        assert_eq!(m.target_kind, AssetKind::Ducklake);
+        assert_eq!(m.target_path, "main");
+        assert!(!m.manual);
+    }
+
+    #[test]
+    fn materialize_manual_only_is_dropped() {
+        // `manual` with no target is not a valid materialization.
+        let out = parse_pipeline_annotations("// materialize manual");
+        assert!(out.materialize.is_none());
+    }
+
+    #[test]
+    fn materialize_first_wins() {
+        let out = parse_pipeline_annotations(
+            "// materialize ducklake://a/x\n# materialize manual ducklake://b/y",
+        );
+        let m = out.materialize.expect("materialize");
+        assert_eq!(m.target_path, "a/x");
+        assert!(!m.manual);
+    }
+
     #[test]
     fn combined() {
         let code = concat!(
@@ -1053,7 +1180,8 @@ mod pipeline_annotation_tests {
             "// partitioned daily tz=\"UTC\"\n",
             "// freshness 2h\n",
             "// tag heavy\n",
-            "// retry 3 5s\n"
+            "// retry 3 5s\n",
+            "// materialize ducklake://analytics/orders_daily key=order_id\n"
         );
         let out = parse_pipeline_annotations(code);
         assert!(out.in_pipeline);
@@ -1064,6 +1192,10 @@ mod pipeline_annotation_tests {
         let r = out.retry.expect("retry");
         assert_eq!(r.count, 3);
         assert_eq!(r.delay.as_deref(), Some("5s"));
+        let m = out.materialize.expect("materialize");
+        assert!(!m.manual);
+        assert_eq!(m.target_path, "analytics/orders_daily");
+        assert_eq!(m.unique_key.as_deref(), Some("order_id"));
     }
 
     #[test]

backend/parsers/windmill-parser/src/lib.rs

@@ -13,6 +13,7 @@ use serde::Serialize;
 use serde_json::Value;
 
 pub mod asset_parser;
+pub mod sql_materialize;
 
 /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types)
 #[derive(Clone, Copy, Debug)]

backend/parsers/windmill-parser/src/sql_materialize.rs

@@ -0,0 +1,817 @@
+//! Eligibility classifier + materialization SQL codegen for managed `// materialize`.
+//!
+//! Managed `// materialize` (the default) promises the script is "setup
+//! statements, then one trailing SELECT" — Windmill generates the write DDL
+//! around that SELECT (the `// materialize manual` escape hatch opts out and
+//! writes its own DDL). This module is the single source of truth for *which
+//! block is that SELECT* and *what DDL gets generated*, so save-time validation
+//! (deploy path) and run-time codegen (DuckDB executor) can never disagree.
+//!
+//! Everything here is pure and string-level: no SQL is executed, no type
+//! inference is done. The classifier is leading-keyword based and deliberately
+//! conservative — anything it can't positively recognize as a read-only output
+//! or a known-safe setup statement is rejected, so a script is only accepted
+//! for managed mode when its shape is unambiguous.
+
+/// One top-level statement's role in a wrap-mode script.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BlockClass {
+    /// Read-only relation the wrap writes from: `SELECT` / `WITH …SELECT` /
+    /// `FROM` (DuckDB from-first) / `VALUES` / `TABLE x` / `(UN)PIVOT`.
+    Output,
+    /// Known-safe preamble: `ATTACH` / `INSTALL` / `LOAD` / `SET` / `PRAGMA` /
+    /// `USE` / `CREATE TEMP …`. Runs verbatim before the generated write.
+    Setup,
+    /// Anything that writes or whose effect we can't vouch for: non-temp
+    /// `CREATE` / `INSERT` / `UPDATE` / `DELETE` / `MERGE` / `DROP` / `COPY` /
+    /// `ALTER` / `TRUNCATE`, or an unrecognized leading keyword. Disqualifies
+    /// managed mode (the user should use `// materialize manual`).
+    Disallowed,
+}
+
+/// A script accepted for wrapping: zero+ setup blocks then one terminal SELECT.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WrapPlan {
+    /// Setup statements in source order, verbatim, **without** trailing `;`.
+    pub setup: Vec<String>,
+    /// The single terminal output statement, verbatim, **without** trailing `;`.
+    pub output: String,
+}
+
+/// Why a script is not eligible for managed `// materialize`. Carries enough to
+/// render the targeted save-time messages.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WrapError {
+    /// No statements at all (empty / comments only).
+    Empty,
+    /// No terminal SELECT — nothing to wrap.
+    NoOutput,
+    /// More than one top-level SELECT. `count` is how many were found.
+    MultipleOutputs { count: usize },
+    /// A SELECT exists but isn't the last statement (something runs after it).
+    OutputNotLast,
+    /// A write/unknown statement appears among the setup blocks. `snippet` is a
+    /// short prefix of the offending statement for the error message.
+    DisallowedBlock { snippet: String },
+}
+
+impl WrapError {
+    /// Human-facing, actionable message (matches the spec's rejection text).
+    pub fn message(&self) -> String {
+        let base =
+            "managed `// materialize` requires the script to be setup statements then a single trailing SELECT";
+        let manual = "use `// materialize manual` to write the DDL yourself";
+        match self {
+            WrapError::Empty => format!("{base}: the script is empty."),
+            WrapError::NoOutput => format!("{base}: found no SELECT — {manual}."),
+            WrapError::MultipleOutputs { count } => format!(
+                "{base}: found {count} SELECT statements; combine them with a CTE, or {manual}."
+            ),
+            WrapError::OutputNotLast => format!(
+                "{base}: found statements after the SELECT — move them above it, or {manual}."
+            ),
+            WrapError::DisallowedBlock { snippet } => {
+                format!("{base}: `{snippet}` writes or is unrecognized — {manual}.")
+            }
+        }
+    }
+}
+
+/// Split SQL into top-level, `;`-separated statements, skipping line comments
+/// (`-- …`), block comments (`/* … */`), single-quoted strings (`'…'` with
+/// `''` escape) and double-quoted identifiers (`"…"`). Semicolons inside any of
+/// those are not separators. Returns each statement trimmed, comments stripped,
+/// empties dropped. Self-contained so the parser crate stays dependency-free;
+/// it must stay behaviourally aligned with the executor's block splitter (both
+/// route wrap through `classify_wrap`, so the split they see is this one).
+pub fn split_statements(sql: &str) -> Vec<String> {
+    let mut out = Vec::new();
+    let mut cur = String::new();
+    let bytes = sql.as_bytes();
+    let mut i = 0;
+    let n = bytes.len();
+    while i < n {
+        let c = bytes[i] as char;
+        // line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it
+        // is how Windmill pipeline annotations (`// materialize`, `// pipeline`,
+        // …) are written, and they sit above the SQL in the same script; strip
+        // them so they don't pollute the first statement block's classification
+        // or the generated setup SQL.
+        if (c == '-' && i + 1 < n && bytes[i + 1] == b'-')
+            || (c == '/' && i + 1 < n && bytes[i + 1] == b'/')
+        {
+            while i < n && bytes[i] != b'\n' {
+                i += 1;
+            }
+            continue;
+        }
+        // block comment
+        if c == '/' && i + 1 < n && bytes[i + 1] == b'*' {
+            i += 2;
+            while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
+                i += 1;
+            }
+            i += 2;
+            continue;
+        }
+        // single-quoted string
+        if c == '\'' {
+            cur.push(c);
+            i += 1;
+            while i < n {
+                cur.push(bytes[i] as char);
+                if bytes[i] == b'\'' {
+                    // doubled '' is an escaped quote, stay in string
+                    if i + 1 < n && bytes[i + 1] == b'\'' {
+                        cur.push('\'');
+                        i += 2;
+                        continue;
+                    }
+                    i += 1;
+                    break;
+                }
+                i += 1;
+            }
+            continue;
+        }
+        // double-quoted identifier
+        if c == '"' {
+            cur.push(c);
+            i += 1;
+            while i < n {
+                cur.push(bytes[i] as char);
+                if bytes[i] == b'"' {
+                    i += 1;
+                    break;
+                }
+                i += 1;
+            }
+            continue;
+        }
+        if c == ';' {
+            let t = cur.trim();
+            if !t.is_empty() {
+                out.push(t.to_string());
+            }
+            cur.clear();
+            i += 1;
+            continue;
+        }
+        cur.push(c);
+        i += 1;
+    }
+    let t = cur.trim();
+    if !t.is_empty() {
+        out.push(t.to_string());
+    }
+    out
+}
+
+/// Lowercased top-level keyword tokens of a single statement (parens collapsed
+/// away: tokens *inside* balanced `(...)` are skipped, so a CTE body's verbs
+/// don't leak up). Strings/identifiers are already gone from the split, but we
+/// re-guard quotes defensively. Used to disambiguate `WITH …` and `CREATE …`.
+fn top_level_keywords(stmt: &str) -> Vec<String> {
+    let mut toks = Vec::new();
+    let mut cur = String::new();
+    let mut depth: i32 = 0;
+    let bytes = stmt.as_bytes();
+    let mut i = 0;
+    let n = bytes.len();
+    let flush = |cur: &mut String, toks: &mut Vec<String>| {
+        if !cur.is_empty() {
+            toks.push(cur.to_lowercase());
+            cur.clear();
+        }
+    };
+    while i < n {
+        let c = bytes[i] as char;
+        if c == '\'' || c == '"' {
+            let q = bytes[i];
+            i += 1;
+            while i < n && bytes[i] != q {
+                i += 1;
+            }
+            i += 1;
+            continue;
+        }
+        if c == '(' {
+            flush(&mut cur, &mut toks);
+            depth += 1;
+            i += 1;
+            continue;
+        }
+        if c == ')' {
+            if depth > 0 {
+                depth -= 1;
+            }
+            i += 1;
+            continue;
+        }
+        if depth > 0 {
+            i += 1;
+            continue;
+        }
+        if c.is_alphanumeric() || c == '_' {
+            cur.push(c);
+        } else {
+            flush(&mut cur, &mut toks);
+        }
+        i += 1;
+    }
+    flush(&mut cur, &mut toks);
+    toks
+}
+
+const OUTPUT_KW: &[&str] = &["select", "from", "values", "table", "pivot", "unpivot"];
+const SETUP_KW: &[&str] = &["attach", "install", "load", "set", "pragma", "use"];
+const WRITE_VERBS: &[&str] = &["insert", "update", "delete", "merge"];
+
+/// Classify a single statement by its leading keyword (with `WITH`/`CREATE`
+/// disambiguation). See [`BlockClass`].
+pub fn classify_block(stmt: &str) -> BlockClass {
+    let kws = top_level_keywords(stmt);
+    let Some(first) = kws.first().map(String::as_str) else {
+        return BlockClass::Disallowed;
+    };
+
+    // CREATE TEMP … is setup (staging); any other CREATE is a write.
+    if first == "create" {
+        let temp = kws
+            .iter()
+            .skip(1)
+            .take(3)
+            .any(|k| k == "temp" || k == "temporary");
+        return if temp {
+            BlockClass::Setup
+        } else {
+            BlockClass::Disallowed
+        };
+    }
+
+    // WITH … : the main statement's verb decides. CTE bodies are parenthesized,
+    // so their verbs are not in `kws`; the first top-level write verb or SELECT
+    // after the CTE list is the real one.
+    if first == "with" {
+        for k in kws.iter().skip(1) {
+            if k == "select" {
+                return BlockClass::Output;
+            }
+            if WRITE_VERBS.contains(&k.as_str()) {
+                return BlockClass::Disallowed;
+            }
+        }
+        // `WITH x AS (...) SELECT` where SELECT got collapsed is impossible
+        // (SELECT here is top-level), so a WITH with no top-level verb is a
+        // malformed/unknown statement — reject conservatively.
+        return BlockClass::Disallowed;
+    }
+
+    if OUTPUT_KW.contains(&first) {
+        return BlockClass::Output;
+    }
+    if SETUP_KW.contains(&first) {
+        return BlockClass::Setup;
+    }
+    BlockClass::Disallowed
+}
+
+/// Validate a script for managed `// materialize` and, on success, return the
+/// setup/output split. Enforces the four conditions from the spec:
+/// 1. exactly one Output block, 2. it is last, 3. all preceding blocks are
+/// Setup, 4. nothing after it.
+pub fn classify_wrap(sql: &str) -> Result<WrapPlan, WrapError> {
+    let stmts = split_statements(sql);
+    if stmts.is_empty() {
+        return Err(WrapError::Empty);
+    }
+    let classes: Vec<BlockClass> = stmts.iter().map(|s| classify_block(s)).collect();
+
+    let output_idxs: Vec<usize> = classes
+        .iter()
+        .enumerate()
+        .filter(|(_, c)| **c == BlockClass::Output)
+        .map(|(i, _)| i)
+        .collect();
+
+    match output_idxs.len() {
+        0 => return Err(WrapError::NoOutput),
+        1 => {}
+        count => return Err(WrapError::MultipleOutputs { count }),
+    }
+    let out_idx = output_idxs[0];
+    if out_idx != stmts.len() - 1 {
+        return Err(WrapError::OutputNotLast);
+    }
+    // Everything before the output must be Setup (no Disallowed preamble).
+    for (i, c) in classes.iter().enumerate().take(out_idx) {
+        if *c != BlockClass::Setup {
+            return Err(WrapError::DisallowedBlock { snippet: snippet(&stmts[i]) });
+        }
+    }
+    Ok(WrapPlan { setup: stmts[..out_idx].to_vec(), output: stmts[out_idx].clone() })
+}
+
+fn snippet(stmt: &str) -> String {
+    let one_line: String = stmt.split_whitespace().collect::<Vec<_>>().join(" ");
+    if one_line.chars().count() > 40 {
+        let truncated: String = one_line.chars().take(40).collect();
+        format!("{truncated}…")
+    } else {
+        one_line
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Codegen
+// ---------------------------------------------------------------------------
+
+/// How a (partition of a) materialized table is reconciled on each run.
+/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else
+/// `unique_key` → `Merge`, else `Replace`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MaterializeStrategy {
+    /// DELETE the current partition, then INSERT — partition becomes exactly
+    /// what the SELECT returned. Full-refresh of the slice.
+    Replace,
+    /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows
+    /// absent from the SELECT are left in place.
+    Merge { unique_key: String },
+    /// INSERT only — immutable event-log semantics.
+    Append,
+}
+
+/// Inputs to materialization codegen, all resolved at run time by the worker.
+/// Pure: produces SQL text; executes nothing.
+#[derive(Debug, Clone)]
+pub struct MaterializeCodegen<'a> {
+    /// Fully-qualified target, e.g. `_wm_target.orders_daily`. Always qualified
+    /// so a user `USE …;` in setup can't redirect the write.
+    pub target_qualified: &'a str,
+    /// The user's output SELECT (verbatim, no trailing `;`) — embedded as a
+    /// subquery so its own shape is irrelevant to the generated wrapper.
+    pub select_sql: &'a str,
+    /// Physical partition column added to the managed table.
+    pub partition_col: &'a str,
+    /// SQL expression for the current partition value — a literal like
+    /// `'2026-06-19'` or a bind placeholder. The caller is responsible for
+    /// safe quoting/binding.
+    pub partition_value_sql: &'a str,
+    /// Whether `// partitioned` applies. When false the table is unpartitioned
+    /// and the partition column / `SET PARTITIONED BY` are omitted.
+    pub partitioned: bool,
+    pub strategy: MaterializeStrategy,
+}
+
+impl<'a> MaterializeCodegen<'a> {
+    /// The ordered statements that perform the materialization, to be run after
+    /// the setup blocks and inside the caller's execution. The first-run
+    /// bootstrap is idempotent (`IF NOT EXISTS`), so this is safe to run every
+    /// time. The DELETE/INSERT body is wrapped in one transaction so a partial
+    /// failure leaves the prior snapshot intact. Every strategy reduces to
+    /// DELETE+INSERT (no `MERGE INTO`) — see the `Merge` arm for why.
+    pub fn statements(&self) -> Vec<String> {
+        let t = self.target_qualified;
+        let sel = self.select_sql;
+        let pcol = self.partition_col;
+        let pval = self.partition_value_sql;
+        let mut out = Vec::new();
+
+        // Whole-table replace: rebuild the table to match the SELECT's *current*
+        // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake
+        // still snapshots). This is the only path that survives a changed SELECT
+        // or a pre-existing table with a different schema — the persist-and-
+        // mutate paths below fix the schema at first create.
+        if !self.partitioned && matches!(self.strategy, MaterializeStrategy::Replace) {
+            out.push(format!(
+                "CREATE OR REPLACE TABLE {t} AS SELECT * FROM ({sel});"
+            ));
+            return out;
+        }
+
+        // Persist-and-mutate (partitioned, or merge/append): bootstrap the table
+        // if absent, then write into it. The schema is fixed at first create —
+        // a later SELECT-schema change needs a manual rebuild (schema evolution
+        // is a follow-up).
+        if self.partitioned {
+            out.push(format!(
+                "CREATE TABLE IF NOT EXISTS {t} AS \
+                 SELECT *, CAST(NULL AS VARCHAR) AS {pcol} FROM ({sel}) WHERE false;"
+            ));
+            out.push(format!("ALTER TABLE {t} SET PARTITIONED BY ({pcol});"));
+        } else {
+            out.push(format!(
+                "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;"
+            ));
+        }
+
+        out.push("BEGIN TRANSACTION;".to_string());
+        // The rows to write, with the partition column appended when partitioned.
+        let source = if self.partitioned {
+            format!("SELECT *, {pval} AS {pcol} FROM ({sel})")
+        } else {
+            format!("SELECT * FROM ({sel})")
+        };
+        match &self.strategy {
+            MaterializeStrategy::Replace => {
+                // Only reached when partitioned (whole-table replace returned above).
+                out.push(format!("DELETE FROM {t} WHERE {pcol} = {pval};"));
+                out.push(format!("INSERT INTO {t} {source};"));
+            }
+            MaterializeStrategy::Append => {
+                out.push(format!("INSERT INTO {t} {source};"));
+            }
+            MaterializeStrategy::Merge { unique_key } => {
+                // Upsert within the slice via delete-by-key + insert (dbt's
+                // `delete+insert`): rows whose key is in the incoming SELECT are
+                // replaced, others are left in place. This deliberately avoids
+                // `MERGE INTO` — DuckLake's MERGE fails writing the first rows of
+                // a fresh partition (HTTP 404 on the new parquet), and a failed
+                // write leaves the table needing a DROP. DELETE+INSERT is the
+                // same write shape as `replace`, which is reliable. The DELETE is
+                // scoped to the current partition when partitioned so it stays
+                // slice-local (a key present in another partition is untouched).
+                let scope = if self.partitioned {
+                    format!("{pcol} = {pval} AND ")
+                } else {
+                    String::new()
+                };
+                out.push(format!(
+                    "DELETE FROM {t} WHERE {scope}{unique_key} IN (SELECT {unique_key} FROM ({sel}));"
+                ));
+                out.push(format!("INSERT INTO {t} {source};"));
+            }
+        }
+        out.push("COMMIT;".to_string());
+        out
+    }
+}
+
+/// The read that captures the DuckLake snapshot id produced by the write, for
+/// the given attach alias (e.g. `_wm_target`). The worker runs this last and
+/// records the result into `materialized_partition`.
+pub fn snapshot_capture_sql(alias: &str) -> String {
+    format!("SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('{alias}');")
+}
+
+/// Reserved attach alias for the materialization target, fully-qualified in all
+/// generated SQL so a user `USE …;` in the setup blocks can't redirect the
+/// write. The worker resolves the real `ATTACH 'ducklake:…' AS _wm_target (…)`
+/// from the target ducklake's config and passes it in as `target_attach`.
+pub const TARGET_ALIAS: &str = "_wm_target";
+
+/// Assemble the full ordered statement list the DuckDB executor runs for a
+/// managed `// materialize` script. This is the single entry point the worker
+/// calls; it composes the already-tested pieces (classifier split → target
+/// ATTACH → strategy codegen → snapshot capture) so their ordering lives in one
+/// tested place rather than inline in the executor.
+///
+/// `target_attach` is the real `ATTACH 'ducklake:…' AS _wm_target (…);` string
+/// the worker built from config (it depends on resolved credentials, so it
+/// can't be generated here). `target_table` is the table within that catalog
+/// (e.g. `orders_daily`), referenced as `_wm_target.<table>`. `asset_path` is
+/// the full `<name>/<table>` for the result summary. The trailing statement is
+/// a one-row summary read (asset / rows / snapshot_id) that is both the job's
+/// result (a useful preview) and what the worker records.
+pub fn build_wrap_blocks(
+    plan: &WrapPlan,
+    target_attach: &str,
+    target_table: &str,
+    asset_path: &str,
+    partition_col: &str,
+    partition_value_sql: &str,
+    partitioned: bool,
+    strategy: MaterializeStrategy,
+) -> Vec<String> {
+    let target_qualified = format!("{TARGET_ALIAS}.{target_table}");
+    let cg = MaterializeCodegen {
+        target_qualified: &target_qualified,
+        select_sql: &plan.output,
+        partition_col,
+        partition_value_sql,
+        partitioned,
+        strategy,
+    };
+    let mut blocks: Vec<String> = Vec::new();
+    // Setup blocks come from the splitter with their `;` stripped — re-terminate
+    // each so that when the executor re-joins and re-splits the assembled query,
+    // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH)
+    // don't merge into one malformed statement.
+    blocks.extend(plan.setup.iter().map(|s| terminate(s)));
+    blocks.push(target_attach.to_string());
+    blocks.extend(cg.statements());
+    blocks.push(materialize_result_sql(
+        &target_qualified,
+        asset_path,
+        partition_col,
+        partition_value_sql,
+        partitioned,
+    ));
+    blocks
+}
+
+/// The trailing one-row summary the materialize run returns: the asset it
+/// produced, the row count of the materialized slice (the partition when
+/// partitioned, else the whole table), and the DuckLake snapshot it created.
+/// This is both a useful preview result and the row the worker records.
+pub fn materialize_result_sql(
+    target_qualified: &str,
+    asset_path: &str,
+    partition_col: &str,
+    partition_value_sql: &str,
+    partitioned: bool,
+) -> String {
+    let (count_expr, partition_sel) = if partitioned {
+        // Row count is the slice this run wrote (the partition); `partition`
+        // lets the UI label the count and scope the preview to it.
+        (
+            format!(
+                "(SELECT count(*) FROM {target_qualified} WHERE {partition_col} = {partition_value_sql})"
+            ),
+            format!("{partition_value_sql} AS partition, "),
+        )
+    } else {
+        (
+            format!("(SELECT count(*) FROM {target_qualified})"),
+            String::new(),
+        )
+    };
+    format!(
+        "SELECT 'ducklake://{asset_path}' AS materialized, \
+         {partition_sel}{count_expr} AS rows, \
+         (SELECT max(snapshot_id) FROM ducklake_snapshots('{TARGET_ALIAS}')) AS snapshot_id;"
+    )
+}
+
+// Ensure a statement ends with a single `;`.
+fn terminate(stmt: &str) -> String {
+    let t = stmt.trim_end();
+    if t.ends_with(';') {
+        t.to_string()
+    } else {
+        format!("{t};")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ok(sql: &str) -> WrapPlan {
+        classify_wrap(sql).expect("expected wrap-eligible")
+    }
+    fn err(sql: &str) -> WrapError {
+        classify_wrap(sql).expect_err("expected wrap-ineligible")
+    }
+
+    #[test]
+    fn split_respects_strings_comments_idents() {
+        let sql = "SET x=1; -- a; comment\nSELECT ';' AS a, \"weird;col\" /* ; */ FROM t;";
+        let s = split_statements(sql);
+        assert_eq!(s.len(), 2);
+        assert_eq!(s[0], "SET x=1");
+        assert!(s[1].starts_with("SELECT"));
+        assert!(s[1].contains("\"weird;col\""));
+    }
+
+    #[test]
+    fn split_handles_escaped_quote() {
+        let s = split_statements("SELECT 'it''s; fine' AS a;");
+        assert_eq!(s.len(), 1);
+        assert!(s[0].contains("it''s; fine"));
+    }
+
+    #[test]
+    fn pipeline_annotations_are_stripped() {
+        // The real shape: `//` annotation lines above the SQL must not pollute
+        // the first block's classification (regression — they were being read
+        // as a leading `pipeline` keyword and rejected).
+        let p = ok("// pipeline\n// materialize ducklake://main/t\n// partitioned daily\nATTACH 'ducklake://main' AS dl;\nSELECT 1 AS id");
+        assert_eq!(p.setup.len(), 1);
+        // The annotation lines are gone — the setup block starts at the real
+        // SQL (the `//` inside `ducklake://main` is legitimately retained).
+        assert!(p.setup[0].starts_with("ATTACH"));
+        assert!(p.output.starts_with("SELECT"));
+    }
+
+    #[test]
+    fn bare_select_is_eligible() {
+        let p = ok("SELECT a, b FROM t WHERE c = '{partition}'");
+        assert!(p.setup.is_empty());
+        assert!(p.output.starts_with("SELECT"));
+    }
+
+    #[test]
+    fn setup_then_select_is_eligible() {
+        let p = ok(
+            "ATTACH 'ducklake://main' AS dl;\n SET memory_limit='4GB';\n SELECT * FROM dl.orders",
+        );
+        assert_eq!(p.setup.len(), 2);
+        assert!(p.output.starts_with("SELECT"));
+    }
+
+    #[test]
+    fn create_temp_staging_is_setup() {
+        let p = ok("CREATE TEMP TABLE s AS SELECT 1; SELECT * FROM s");
+        assert_eq!(p.setup.len(), 1);
+        assert_eq!(
+            classify_block("CREATE TEMP TABLE s AS SELECT 1"),
+            BlockClass::Setup
+        );
+        assert_eq!(
+            classify_block("CREATE OR REPLACE TEMPORARY VIEW v AS SELECT 1"),
+            BlockClass::Setup
+        );
+    }
+
+    #[test]
+    fn with_cte_select_is_output_write_is_disallowed() {
+        assert_eq!(
+            classify_block("WITH x AS (SELECT 1) SELECT * FROM x"),
+            BlockClass::Output
+        );
+        // CTE whose main statement inserts is a write, even though it starts WITH.
+        assert_eq!(
+            classify_block("WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x"),
+            BlockClass::Disallowed
+        );
+    }
+
+    #[test]
+    fn from_first_and_values_are_output() {
+        assert_eq!(classify_block("FROM t SELECT a"), BlockClass::Output);
+        assert_eq!(classify_block("VALUES (1),(2)"), BlockClass::Output);
+        assert_eq!(classify_block("TABLE t"), BlockClass::Output);
+    }
+
+    #[test]
+    fn trailing_write_rejected() {
+        assert_eq!(
+            err("SELECT * FROM t; INSERT INTO u VALUES (1)"),
+            WrapError::OutputNotLast
+        );
+    }
+
+    #[test]
+    fn write_in_preamble_rejected() {
+        match err("INSERT INTO t VALUES (1); SELECT * FROM t") {
+            WrapError::DisallowedBlock { snippet } => assert!(snippet.starts_with("INSERT")),
+            e => panic!("wrong error: {e:?}"),
+        }
+    }
+
+    #[test]
+    fn multiple_selects_rejected() {
+        assert_eq!(
+            err("SELECT 1; SELECT 2"),
+            WrapError::MultipleOutputs { count: 2 }
+        );
+    }
+
+    #[test]
+    fn no_select_and_empty_rejected() {
+        assert_eq!(err("CREATE TABLE t (a INT)"), WrapError::NoOutput);
+        assert_eq!(err("   -- just a comment\n"), WrapError::Empty);
+    }
+
+    #[test]
+    fn use_cannot_redirect_is_classified_setup() {
+        // `USE` is allowed setup; generated SQL is fully qualified regardless.
+        assert_eq!(classify_block("USE dl"), BlockClass::Setup);
+    }
+
+    #[test]
+    fn codegen_replace_partitioned() {
+        let cg = MaterializeCodegen {
+            target_qualified: "_wm_target.orders_daily",
+            select_sql: "SELECT a FROM dl.orders",
+            partition_col: "_wm_partition",
+            partition_value_sql: "'2026-06-19'",
+            partitioned: true,
+            strategy: MaterializeStrategy::Replace,
+        };
+        let st = cg.statements();
+        assert!(st[0].contains("CREATE TABLE IF NOT EXISTS _wm_target.orders_daily"));
+        assert!(st[0].contains("CAST(NULL AS VARCHAR) AS _wm_partition"));
+        assert!(st.iter().any(
+            |s| s == "ALTER TABLE _wm_target.orders_daily SET PARTITIONED BY (_wm_partition);"
+        ));
+        assert!(st.iter().any(|s| s.starts_with(
+            "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'"
+        )));
+        assert!(st.iter().any(|s| s.contains(
+            "INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19' AS _wm_partition"
+        )));
+        assert_eq!(st.first().map(|_| &st[st.len() - 1]).unwrap(), "COMMIT;");
+    }
+
+    #[test]
+    fn codegen_merge_is_delete_by_key_plus_insert() {
+        let cg = MaterializeCodegen {
+            target_qualified: "_wm_target.orders_daily",
+            select_sql: "SELECT order_id, amount FROM dl.orders",
+            partition_col: "_wm_partition",
+            partition_value_sql: "'2026-06-19'",
+            partitioned: true,
+            strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() },
+        };
+        let st = cg.statements();
+        // upsert = delete-by-key (partition-scoped) + insert — NO `MERGE INTO`
+        // (DuckLake's MERGE fails on fresh partitions).
+        assert!(!st.iter().any(|s| s.contains("MERGE INTO")));
+        let del = st
+            .iter()
+            .find(|s| s.starts_with("DELETE FROM"))
+            .expect("delete stmt");
+        assert!(del.contains(
+            "WHERE _wm_partition = '2026-06-19' AND order_id IN (SELECT order_id FROM (SELECT order_id, amount FROM dl.orders))"
+        ));
+        assert!(st
+            .iter()
+            .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'")));
+    }
+
+    #[test]
+    fn codegen_append_inserts_only() {
+        let cg = MaterializeCodegen {
+            target_qualified: "_wm_target.events",
+            select_sql: "SELECT * FROM dl.raw",
+            partition_col: "_wm_partition",
+            partition_value_sql: "'2026-06-19'",
+            partitioned: true,
+            strategy: MaterializeStrategy::Append,
+        };
+        let st = cg.statements();
+        assert!(st
+            .iter()
+            .any(|s| s.starts_with("INSERT INTO _wm_target.events")));
+        assert!(!st.iter().any(|s| s.starts_with("DELETE")));
+        assert!(!st.iter().any(|s| s.starts_with("MERGE")));
+    }
+
+    #[test]
+    fn codegen_whole_table_replace_is_create_or_replace() {
+        // Unpartitioned replace must use CREATE OR REPLACE so a changed SELECT
+        // schema (or a pre-existing table with a different schema) doesn't break
+        // — and nothing else (no bootstrap / DELETE / INSERT / txn).
+        let cg = MaterializeCodegen {
+            target_qualified: "_wm_target.customer_dim",
+            select_sql: "SELECT a, b, c FROM dl.src",
+            partition_col: "_wm_partition",
+            partition_value_sql: "''",
+            partitioned: false,
+            strategy: MaterializeStrategy::Replace,
+        };
+        let st = cg.statements();
+        assert_eq!(
+            st,
+            vec![
+                "CREATE OR REPLACE TABLE _wm_target.customer_dim AS SELECT * FROM (SELECT a, b, c FROM dl.src);"
+                    .to_string()
+            ]
+        );
+    }
+
+    #[test]
+    fn snapshot_capture_targets_alias() {
+        assert_eq!(
+            snapshot_capture_sql("_wm_target"),
+            "SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('_wm_target');"
+        );
+    }
+
+    #[test]
+    fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() {
+        let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'");
+        let blocks = build_wrap_blocks(
+            &plan,
+            "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');",
+            "orders_daily",
+            "main/orders_daily",
+            "_wm_partition",
+            "'2026-06-19'",
+            true,
+            MaterializeStrategy::Replace,
+        );
+        // setup block first, then the target ATTACH, then codegen, then result.
+        assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl"));
+        // every setup block must be `;`-terminated so re-splitting can't merge it
+        // with the synthetic target ATTACH that follows.
+        assert!(blocks[0].ends_with(';'));
+        assert_eq!(
+            blocks[1],
+            "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');"
+        );
+        assert!(blocks.iter().any(|b| b.contains("_wm_target.orders_daily")));
+        assert!(blocks.iter().any(|b| b.starts_with(
+            "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'"
+        )));
+        // the trailing block is the one-row summary (asset / rows / snapshot_id),
+        // partition-scoped for the row count
+        let last = blocks.last().unwrap();
+        assert!(last.contains("'ducklake://main/orders_daily' AS materialized"));
+        assert!(last.contains("'2026-06-19' AS partition"));
+        assert!(last.contains("WHERE _wm_partition = '2026-06-19') AS rows"));
+        assert!(last.contains("ducklake_snapshots('_wm_target')"));
+    }
+}

backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json

@@ -234,5 +234,72 @@
 			"tag": null,
 			"retry": null
 		}
+	},
+	{
+		"name": "materialize managed (default) with merge key",
+		"code": "// pipeline\n// materialize ducklake://analytics/orders_daily key=order_id\nSELECT 1;",
+		"expected": {
+			"in_pipeline": true,
+			"asset_triggers": [],
+			"native_triggers": [],
+			"partition": null,
+			"freshness": null,
+			"tag": null,
+			"retry": null,
+			"materialize": {
+				"target_kind": "ducklake",
+				"target_path": "analytics/orders_daily",
+				"unique_key": "order_id"
+			}
+		}
+	},
+	{
+		"name": "materialize manual escape hatch, first value wins",
+		"code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}",
+		"expected": {
+			"in_pipeline": false,
+			"asset_triggers": [],
+			"native_triggers": [],
+			"partition": null,
+			"freshness": null,
+			"tag": null,
+			"retry": null,
+			"materialize": {
+				"target_kind": "ducklake",
+				"target_path": "analytics/orders_daily",
+				"manual": true
+			}
+		}
+	},
+	{
+		"name": "materialize default-syntax shorthand with append",
+		"code": "// materialize ducklake append\nexport function main() {}",
+		"expected": {
+			"in_pipeline": false,
+			"asset_triggers": [],
+			"native_triggers": [],
+			"partition": null,
+			"freshness": null,
+			"tag": null,
+			"retry": null,
+			"materialize": {
+				"target_kind": "ducklake",
+				"target_path": "main",
+				"append": true
+			}
+		}
+	},
+	{
+		"name": "materialize manual with no target is dropped",
+		"code": "// materialize manual\nexport function main() {}",
+		"expected": {
+			"in_pipeline": false,
+			"asset_triggers": [],
+			"native_triggers": [],
+			"partition": null,
+			"freshness": null,
+			"tag": null,
+			"retry": null
+		}
 	}
 ]

backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs

@@ -34,6 +34,22 @@ struct Expected {
     freshness: Option<String>,
     tag: Option<String>,
     retry: Option<ExpectedRetry>,
+    // Default-on-absent so the pre-existing fixtures (which omit it) keep
+    // deserializing; only fixtures exercising materialization set it.
+    #[serde(default)]
+    materialize: Option<ExpectedMaterialize>,
+}
+
+#[derive(Deserialize)]
+struct ExpectedMaterialize {
+    target_kind: String,
+    target_path: String,
+    #[serde(default)]
+    manual: bool,
+    #[serde(default)]
+    append: bool,
+    #[serde(default)]
+    unique_key: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -153,5 +169,25 @@ fn pipeline_annotation_fixtures_match() {
                 want.is_some()
             ),
         }
+
+        match (&got.materialize, &f.expected.materialize) {
+            (None, None) => {}
+            (Some(m), Some(e)) => {
+                assert_eq!(
+                    kind_str(m.target_kind),
+                    e.target_kind,
+                    "{ctx}: materialize kind"
+                );
+                assert_eq!(m.target_path, e.target_path, "{ctx}: materialize path");
+                assert_eq!(m.manual, e.manual, "{ctx}: materialize manual");
+                assert_eq!(m.append, e.append, "{ctx}: materialize append");
+                assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key");
+            }
+            (got, want) => panic!(
+                "{ctx}: materialize mismatch — got {:?}, want present={}",
+                got,
+                want.is_some()
+            ),
+        }
     }
 }

backend/windmill-api-assets/src/lib.rs

@@ -22,6 +22,63 @@ pub fn workspaced_service() -> Router {
         .route("/list_favorites", get(list_favorites))
         .route("/graph", get(asset_graph))
         .route("/pipelines", get(list_pipeline_folders))
+        .route("/partitions", get(list_partitions))
+        .route("/record_materialization", post(record_materialization))
+}
+
+#[derive(Deserialize)]
+struct PartitionsQuery {
+    // The materialized asset path (`<ducklake>/<table>`).
+    path: String,
+}
+
+// Per-partition materialization status for a ducklake asset — drives the
+// partition-status grid and the backfill worklist. Materialization targets are
+// ducklake-only in v1, so the kind is fixed.
+async fn list_partitions(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+    Query(q): Query<PartitionsQuery>,
+) -> JsonResult<Vec<windmill_common::materialization::MaterializedPartition>> {
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = windmill_common::materialization::list_materialized_partitions(
+        &mut *tx,
+        &w_id,
+        AssetKind::Ducklake,
+        &q.path,
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(Json(rows))
+}
+
+// Record a materialization outcome from a polyglot (Python/TS) `wmill.ducklake`
+// helper running as a pipeline step. The DuckDB `// materialize` engine records
+// this itself; the SDK helpers post here instead so SDK-materialized slices show
+// up in the grid identically. RLS-scoped to the caller's workspace.
+async fn record_materialization(
+    authed: ApiAuthed,
+    Path(w_id): Path<String>,
+    Extension(user_db): Extension<UserDB>,
+    Json(req): Json<windmill_common::materialization::RecordMaterializationRequest>,
+) -> JsonResult<()> {
+    let mut tx = user_db.begin(&authed).await?;
+    windmill_common::materialization::record_materialization(
+        &mut *tx,
+        &w_id,
+        req.asset_kind,
+        &req.asset_path,
+        &req.partition,
+        req.status,
+        req.snapshot_id,
+        req.row_count,
+        req.job_id,
+        req.error.as_deref(),
+    )
+    .await?;
+    tx.commit().await?;
+    Ok(Json(()))
 }
 
 #[derive(Deserialize)]

backend/windmill-api-scripts/Cargo.toml

@@ -26,6 +26,7 @@ windmill-parser-ts.workspace = true
 windmill-parser.workspace = true
 windmill-parser-ts-asset.workspace = true
 windmill-parser-sql-asset.workspace = true
+windmill-parser-sql.workspace = true
 windmill-parser-yaml.workspace = true
 
 axum.workspace = true

backend/windmill-api-scripts/src/scripts.rs

@@ -1251,6 +1251,72 @@ async fn create_script_internal<'c>(
             windmill_common::pipeline_advanced::freshness_enforcement_todo()
         );
     }
+    // `// materialize` materializes a `ducklake://<name>/<table>` target from a
+    // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake
+    // target would otherwise deploy, register a producer in the asset graph, then
+    // silently no-op at run time (`build_materialized_query` returns `Ok(None)`),
+    // and a non-DuckDB script never reaches the executor that records state. The
+    // managed-only checks (single trailing SELECT, no SQL args) come after — a
+    // `manual` script owns its DDL and skips them.
+    if let Some(m) = pipeline_annotations.materialize.as_ref() {
+        if ns.language != ScriptLang::DuckDb {
+            return Err(Error::BadRequest(format!(
+                "`// materialize` is only supported for DuckDB scripts, not {}. Use the \
+                 wmll.ducklake helpers to materialize from other languages.",
+                ns.language.as_str()
+            )));
+        }
+        if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake {
+            return Err(Error::BadRequest(
+                "`// materialize` only supports a DuckLake target \
+                 (`ducklake://<name>/<table>`); other asset kinds aren't materializable."
+                    .to_string(),
+            ));
+        }
+        if !m.target_path.contains('/') {
+            return Err(Error::BadRequest(format!(
+                "`// materialize` needs a table in the target: \
+                 `ducklake://{0}/<table>` (got `ducklake://{0}`).",
+                m.target_path
+            )));
+        }
+        if !m.manual {
+            if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) {
+                return Err(Error::BadRequest(e.message()));
+            }
+            // Managed materialize strips line comments when it wraps the SELECT,
+            // so a `-- $name (TYPE)` declaration is lost while its `$name`
+            // reference survives in the embedded SELECT — it would run unbound.
+            // Managed materialize takes no SQL args (the partition is supplied by
+            // the engine, not bound). Reject declared args with a clear error.
+            if let Ok(sig) = windmill_parser_sql::parse_duckdb_sig(&ns.content) {
+                if !sig.args.is_empty() {
+                    let names = sig
+                        .args
+                        .iter()
+                        .map(|a| format!("${}", a.name))
+                        .collect::<Vec<_>>()
+                        .join(", ");
+                    return Err(Error::BadRequest(format!(
+                        "managed `// materialize` cannot take SQL arguments ({names}): wrapping your \
+                         SELECT drops the `-- $arg` declarations, so they would run unbound. The \
+                         partition is supplied by the engine — reference its value with the \
+                         `{{partition}}` token, or use `// materialize manual` to write the DDL (and \
+                         bind args) yourself."
+                    )));
+                }
+            }
+        }
+        // `key=` (merge) and `append` are mutually exclusive reconciliation
+        // strategies; append (INSERT-only) wins. Surface the conflict rather
+        // than silently dropping the dedup the author may have intended.
+        if m.unique_key.is_some() && m.append {
+            tracing::warn!(
+                "script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)",
+                ns.path
+            );
+        }
+    }
     let in_pipeline = pipeline_annotations.in_pipeline;
     // `// trigger all` → AND join barrier (else OR, the default).
     let pipeline_join_all = !pipeline_annotations.join_mode.is_any();
@@ -1290,6 +1356,26 @@ async fn create_script_internal<'c>(
         &ns.content,
         ns.assets.take(),
     );
+    // Register the `// materialize` target as a write asset so the deployed
+    // asset graph shows this script as the producer of the managed table — the
+    // body's `SELECT` doesn't express the write (the runtime generates it), so
+    // server-side inference wouldn't otherwise link it.
+    let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() {
+        let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind);
+        let mut a = effective_assets.unwrap_or_default();
+        if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) {
+            a.push(windmill_common::assets::AssetWithAltAccessType {
+                path: m.target_path.clone(),
+                kind,
+                access_type: Some(windmill_common::assets::AssetUsageAccessType::W),
+                alt_access_type: None,
+                columns: None,
+            });
+        }
+        Some(a)
+    } else {
+        effective_assets
+    };
     let auto_kind = if in_pipeline {
         Some("pipeline".to_string())
     } else if ci_test_refs.is_some() {

backend/windmill-api/openapi.yaml

@@ -22576,6 +22576,13 @@ components:
           type: string
         parent_hash:
           type: string
+        auto_parent:
+          type: boolean
+          description: >-
+            When true, the backend resolves the parent to the current deployed
+            head for this path within the transaction (ignoring parent_hash),
+            instead of failing with a "lineage must be linear" error when the
+            supplied parent_hash is stale.
         summary:
           type: string
         description:

backend/windmill-common/src/lib.rs

@@ -61,6 +61,7 @@ pub mod indexer;
 pub mod instance_config;
 pub mod job_metrics;
 pub mod log_context;
+pub mod materialization;
 pub mod min_version;
 pub mod notify_events;
 pub mod runtime_assets;

backend/windmill-common/src/materialization.rs

@@ -0,0 +1,135 @@
+//! CE materialization state — the per-partition status recorded by the managed
+//! `// materialize` write (in windmill-worker), read by the partition-status
+//! grid and by the EE backfill worklist.
+//!
+//! The write engine and this state are CE; only automatic partition
+//! *resolution* (`partition_ee`) and *backfill* orchestration
+//! (`pipeline_advanced_ee`) are enterprise. This module is the shared seam:
+//! the EE backfill enumerates the partitions in a range, diffs them against
+//! these rows to find the missing/failed set, and pushes one CE materialization
+//! job per gap (with an explicit `partition` arg — which runs idempotently and
+//! upserts the row here). Nothing about that orchestration lives in this file;
+//! it only needs the rows to exist, which is why recording is CE.
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use sqlx::PgExecutor;
+use uuid::Uuid;
+
+use crate::assets::AssetKind;
+use crate::error::Result;
+
+/// Sentinel `partition` value for an unpartitioned (whole-table)
+/// materialization — partition is part of the primary key and cannot be NULL.
+pub const UNPARTITIONED: &str = "";
+
+/// Mirrors the `MATERIALIZATION_STATUS` pg enum (see migration
+/// `20260619170118_add_materialized_partition`).
+#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[sqlx(type_name = "MATERIALIZATION_STATUS", rename_all = "lowercase")]
+#[serde(rename_all = "lowercase")]
+pub enum MaterializationStatus {
+    Running,
+    Materialized,
+    Failed,
+}
+
+/// The materialization outcome an agent worker (`Connection::Http`, no direct
+/// DB) sends to the API to be recorded. Mirrors the `record_materialization`
+/// args; the API handler unpacks it and calls that function with its own DB.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RecordMaterializationRequest {
+    pub asset_kind: AssetKind,
+    pub asset_path: String,
+    pub partition: String,
+    pub status: MaterializationStatus,
+    pub snapshot_id: Option<i64>,
+    pub row_count: Option<i64>,
+    pub job_id: Option<Uuid>,
+    pub error: Option<String>,
+}
+
+/// Upsert the latest materialization state for one (asset, partition) slice.
+/// The worker records the terminal outcome once the write finishes:
+/// `Materialized` (with the DuckLake `snapshot_id` + `row_count`) or `Failed`
+/// (with `error`). `Running` mirrors the pg enum but has no writer in this flow.
+/// Idempotent: re-running the same partition overwrites the row — exactly the
+/// backfill / failure-recovery contract.
+#[allow(clippy::too_many_arguments)]
+pub async fn record_materialization<'e>(
+    executor: impl PgExecutor<'e>,
+    workspace_id: &str,
+    asset_kind: AssetKind,
+    asset_path: &str,
+    partition: &str,
+    status: MaterializationStatus,
+    snapshot_id: Option<i64>,
+    row_count: Option<i64>,
+    job_id: Option<Uuid>,
+    error: Option<&str>,
+) -> Result<()> {
+    sqlx::query!(
+        "INSERT INTO materialized_partition
+           (workspace_id, asset_kind, asset_path, partition, status,
+            snapshot_id, row_count, job_id, materialized_at, error)
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)
+         ON CONFLICT (workspace_id, asset_kind, asset_path, partition)
+         DO UPDATE SET status = EXCLUDED.status,
+                       snapshot_id = EXCLUDED.snapshot_id,
+                       row_count = EXCLUDED.row_count,
+                       job_id = EXCLUDED.job_id,
+                       materialized_at = now(),
+                       error = EXCLUDED.error",
+        workspace_id,
+        asset_kind as AssetKind,
+        asset_path,
+        partition,
+        status as MaterializationStatus,
+        snapshot_id,
+        row_count,
+        job_id,
+        error,
+    )
+    .execute(executor)
+    .await?;
+    Ok(())
+}
+
+/// One materialized-partition row, for the status grid / backfill diff.
+#[derive(sqlx::FromRow, Debug, Clone, Serialize)]
+pub struct MaterializedPartition {
+    pub asset_kind: AssetKind,
+    pub asset_path: String,
+    pub partition: String,
+    pub status: MaterializationStatus,
+    pub snapshot_id: Option<i64>,
+    pub row_count: Option<i64>,
+    pub job_id: Option<Uuid>,
+    pub materialized_at: DateTime<Utc>,
+    pub error: Option<String>,
+}
+
+/// All recorded partitions for one asset, newest first — the grid's data and
+/// the backfill worklist's "what already exists" set.
+pub async fn list_materialized_partitions<'e>(
+    executor: impl PgExecutor<'e>,
+    workspace_id: &str,
+    asset_kind: AssetKind,
+    asset_path: &str,
+) -> Result<Vec<MaterializedPartition>> {
+    let rows = sqlx::query_as!(
+        MaterializedPartition,
+        r#"SELECT asset_kind AS "asset_kind: AssetKind", asset_path, partition,
+                  status AS "status: MaterializationStatus", snapshot_id,
+                  row_count, job_id, materialized_at, error
+             FROM materialized_partition
+            WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3
+            ORDER BY partition DESC"#,
+        workspace_id,
+        asset_kind as AssetKind,
+        asset_path,
+    )
+    .fetch_all(executor)
+    .await?;
+    Ok(rows)
+}

backend/windmill-worker/src/agent_workers.rs

@@ -78,4 +78,22 @@ pub async fn get_datatable_resource_from_agent_http(
         .await
 }
 
+/// Record a materialization outcome from an agent worker (no direct DB) via the
+/// API, so `materialized_partition` state lands the same as on a Sql worker.
+// Only called from the duckdb executor, which is itself `#[cfg(feature = "duckdb")]`.
+#[cfg(feature = "duckdb")]
+pub async fn record_materialization_from_agent_http(
+    client: &HttpClient,
+    w_id: &str,
+    req: &windmill_common::materialization::RecordMaterializationRequest,
+) -> anyhow::Result<()> {
+    client
+        .post(
+            &format!("/api/w/{}/agent_workers/record_materialization", w_id),
+            None,
+            req,
+        )
+        .await
+}
+
 pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping";

backend/windmill-worker/src/duckdb_executor.rs

@@ -32,6 +32,178 @@ use crate::sql_utils::remove_comments;
 use windmill_common::client::AuthedClient;
 use windmill_object_store::DEFAULT_STORAGE;
 
+// What a `// materialize` run records into `materialized_partition` once it
+// finishes. `asset_path` is the full `<name>/<table>` (the asset identity);
+// `partition` is "" for an unpartitioned (whole-table) materialization.
+struct MaterializeExec {
+    asset_kind: windmill_common::assets::AssetKind,
+    asset_path: String,
+    partition: String,
+}
+
+// If `query` declares `// materialize <ducklake>`, return what to record plus,
+// for the default managed mode, the rewritten managed-write SQL (in `manual`
+// mode the script writes its own DDL, so the rewrite is `None`). The rewritten
+// SQL contains a synthetic `ATTACH 'ducklake://<name>' AS _wm_target` that the
+// normal ATTACH-transform pass resolves to real credentials — the same path as
+// the user's own ATTACH. Returns `None` when there is no materialize annotation
+// or the target isn't a ducklake (only ducklake is materialized in v1).
+fn build_materialized_query(
+    query: &str,
+    partition_value: Option<&str>,
+) -> Result<Option<(Option<String>, MaterializeExec)>> {
+    use windmill_parser::asset_parser::{parse_pipeline_annotations, AssetKind as PAssetKind};
+    use windmill_parser::sql_materialize::{
+        build_wrap_blocks, classify_wrap, MaterializeStrategy, TARGET_ALIAS,
+    };
+
+    let ann = parse_pipeline_annotations(query);
+    let Some(m) = ann.materialize else {
+        return Ok(None);
+    };
+    if m.target_kind != PAssetKind::Ducklake {
+        return Ok(None);
+    }
+    let partitioned = ann.partition.is_some();
+    let partition = partition_value.unwrap_or("").to_string();
+    // Partition *resolution* is enterprise; in its absence a partitioned
+    // materialize only runs with an explicit `partition` arg. Fail loudly rather
+    // than silently materialize the wrong (empty) slice.
+    if partitioned && partition.is_empty() {
+        return Err(Error::ExecutionErr(
+            "materialize: a `// partitioned` script ran with no resolved partition — pass an \
+             explicit `partition` arg, or enable enterprise partition resolution"
+                .to_string(),
+        ));
+    }
+    // Convention: `ducklake://<name>/<table>` — <name> is the configured
+    // ducklake (resolved like a user ATTACH), <table> is the rest.
+    let (ducklake_name, table) = m
+        .target_path
+        .split_once('/')
+        .unwrap_or((m.target_path.as_str(), ""));
+    let meta = MaterializeExec {
+        asset_kind: windmill_common::assets::AssetKind::Ducklake,
+        asset_path: m.target_path.clone(),
+        partition: partition.clone(),
+    };
+
+    if m.manual {
+        // Escape hatch: the script owns its DDL; we only record state.
+        return Ok(Some((None, meta)));
+    }
+    if table.is_empty() {
+        return Err(Error::ExecutionErr(format!(
+            "materialize: target `ducklake://{}` has no table (use ducklake://<name>/<table>)",
+            m.target_path
+        )));
+    }
+    let mut plan = classify_wrap(query).map_err(|e| Error::ExecutionErr(e.message()))?;
+    // Resolve the `{partition}` token (same token `// on` asset URIs use) to the
+    // current partition value everywhere in the managed script, so a partitioned
+    // materialize can filter its source by the active slice, e.g.
+    // `WHERE day = {partition}`. The token is always replaced by a *complete*
+    // escaped SQL literal (`'…'` with `'` doubled) whether or not the author
+    // quoted it — so a run caller can't pass metacharacters that break out of
+    // the literal and alter statement boundaries. The pre-quoted form
+    // `'{partition}'` is matched first so it doesn't become `''…''`. Only
+    // meaningful when partitioned.
+    if partitioned {
+        let lit = format!("'{}'", partition.replace('\'', "''"));
+        let tok = windmill_common::assets::PARTITION_TOKEN;
+        let quoted_tok = format!("'{tok}'");
+        plan.output = plan.output.replace(&quoted_tok, &lit).replace(tok, &lit);
+        for s in plan.setup.iter_mut() {
+            *s = s.replace(&quoted_tok, &lit).replace(tok, &lit);
+        }
+    }
+    let strategy = if m.append {
+        MaterializeStrategy::Append
+    } else if let Some(uk) = m.unique_key {
+        MaterializeStrategy::Merge { unique_key: uk }
+    } else {
+        MaterializeStrategy::Replace
+    };
+    // Inline the partition as an escaped SQL literal (DuckLake has no bind for
+    // the partition column in our generated DDL).
+    let pval = format!("'{}'", partition.replace('\'', "''"));
+    let synthetic_attach = format!("ATTACH 'ducklake://{ducklake_name}' AS {TARGET_ALIAS};");
+    let blocks = build_wrap_blocks(
+        &plan,
+        &synthetic_attach,
+        table,
+        &m.target_path,
+        "_wm_partition",
+        &pval,
+        partitioned,
+        strategy,
+    );
+    Ok(Some((Some(blocks.join("\n")), meta)))
+}
+
+// Pull a named i64 field (`snapshot_id` / `rows`) out of the trailing summary
+// read — which in wrap mode is the job result. Shape-tolerant (object / array /
+// nested), returns None if absent (literal mode, or capture failed).
+fn extract_i64(result: &RawValue, field: &str) -> Option<i64> {
+    fn find(v: &Value, field: &str) -> Option<i64> {
+        match v {
+            Value::Number(n) => n.as_i64(),
+            Value::Object(m) => m.get(field).and_then(|x| find(x, field)),
+            Value::Array(a) => a.iter().find_map(|x| find(x, field)),
+            _ => None,
+        }
+    }
+    find(&serde_json::from_str::<Value>(result.get()).ok()?, field)
+}
+
+// Best-effort record of a materialization outcome. On a Sql connection it writes
+// the row directly; on an agent worker (Http, no direct DB) it posts to the API
+// so state lands the same way. Never fails the job — a lost row degrades the
+// grid, not the run.
+async fn record_mat(
+    conn: &Connection,
+    w_id: &str,
+    job_id: Uuid,
+    meta: &MaterializeExec,
+    status: windmill_common::materialization::MaterializationStatus,
+    snapshot_id: Option<i64>,
+    row_count: Option<i64>,
+    error: Option<&str>,
+) {
+    let req = windmill_common::materialization::RecordMaterializationRequest {
+        asset_kind: meta.asset_kind,
+        asset_path: meta.asset_path.clone(),
+        partition: meta.partition.clone(),
+        status,
+        snapshot_id,
+        row_count,
+        job_id: Some(job_id),
+        error: error.map(|e| e.to_string()),
+    };
+    let res: anyhow::Result<()> = match conn {
+        Connection::Sql(db) => windmill_common::materialization::record_materialization(
+            db,
+            w_id,
+            req.asset_kind,
+            &req.asset_path,
+            &req.partition,
+            req.status,
+            req.snapshot_id,
+            req.row_count,
+            req.job_id,
+            req.error.as_deref(),
+        )
+        .await
+        .map_err(|e| anyhow::anyhow!("{e:#}")),
+        Connection::Http(client) => {
+            crate::agent_workers::record_materialization_from_agent_http(client, w_id, &req).await
+        }
+    };
+    if let Err(e) = res {
+        tracing::warn!("failed to record materialization state: {e:#}");
+    }
+}
+
 pub async fn do_duckdb(
     job: &MiniPulledJob,
     client: &AuthedClient,
@@ -68,6 +240,30 @@ pub async fn do_duckdb(
         let mut hidden_passwords = hidden_passwords.clone();
         let mut bigquery_credentials = None;
 
+        // Materialization (`// materialize`): rewrite a wrap script into managed
+        // DDL (its synthetic target ATTACH is resolved by the transform pass
+        // below, like the user's own ATTACH); a literal script is left as-is.
+        // `materialize` also carries what to record once the run finishes.
+        let partition_value: Option<String> = job
+            .args
+            .as_ref()
+            .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG))
+            .and_then(|rv| serde_json::from_str::<String>(rv.get()).ok())
+            .filter(|s| !s.is_empty());
+        let materialize = if query.contains("materialize") {
+            build_materialized_query(query, partition_value.as_deref())?
+        } else {
+            None
+        };
+        let materialized_query;
+        let query: &str = match &materialize {
+            Some((Some(rewritten), _)) => {
+                materialized_query = rewritten.clone();
+                &materialized_query
+            }
+            _ => query,
+        };
+
         let sig = parse_duckdb_sig(query)?.args;
         let mut job_args = build_args_values(job, client, conn).await?;
 
@@ -199,6 +395,19 @@ pub async fn do_duckdb(
         let (result, column_order) = match result {
             Ok(r) => r,
             Err(e) => {
+                if let Some((_, meta)) = &materialize {
+                    record_mat(
+                        conn,
+                        &job.workspace_id,
+                        job.id,
+                        meta,
+                        windmill_common::materialization::MaterializationStatus::Failed,
+                        None,
+                        None,
+                        Some(&e.to_string()),
+                    )
+                    .await;
+                }
                 if let Some(s3_proxy_err) = S3_PROXY_LAST_ERRORS_CACHE.get(&client.token) {
                     return Err(Error::ExecutionErr(format!(
                         "{}\n\nS3 Related Error: {}",
@@ -210,6 +419,24 @@ pub async fn do_duckdb(
             }
         };
 
+        if let Some((_, meta)) = &materialize {
+            // In wrap mode the job result is the summary read (snapshot_id +
+            // rows); in literal mode there is none, so both stay None.
+            let snapshot_id = extract_i64(&result, "snapshot_id");
+            let row_count = extract_i64(&result, "rows");
+            record_mat(
+                conn,
+                &job.workspace_id,
+                job.id,
+                meta,
+                windmill_common::materialization::MaterializationStatus::Materialized,
+                snapshot_id,
+                row_count,
+                None,
+            )
+            .await;
+        }
+
         drop(bigquery_credentials);
 
         *column_order_ref = column_order;

backend/windmill-worker/src/worker.rs

@@ -4324,20 +4324,25 @@ async fn resolve_partition_for_job(
     job: &MiniPulledJob,
     code: &str,
     conn: &Connection,
-) -> error::Result<Option<MiniPulledJob>> {
+) -> error::Result<(Option<MiniPulledJob>, bool)> {
     use windmill_common::partition::{resolve_partition, PARTITION_ARG};
     use windmill_parser::asset_parser::PartitionKind;
 
-    // Only deployed scripts participate in asset pipelines. Cheap
-    // substring guard so the overwhelming majority of script jobs (no
-    // `// partitioned` line) skip the full annotation scan on the hot
-    // path; a false positive only costs one extra parse, never wrong.
-    if !matches!(job.kind, JobKind::Script) || !code.contains("partitioned") {
-        return Ok(None);
+    // Only deployed scripts participate in asset pipelines. Cheap substring
+    // guard so the overwhelming majority of script jobs skip the annotation
+    // scan; when one might be present we parse *once* here and reuse the result
+    // for both `in_pipeline` (→ WM_PIPELINE env, read by the wmll.ducklake SDK to
+    // record state) and `partition` resolution — no second parse downstream. The
+    // bool is whether the script is a `// pipeline` member.
+    if !matches!(job.kind, JobKind::Script)
+        || !(code.contains("pipeline") || code.contains("partitioned"))
+    {
+        return Ok((None, false));
     }
-    let Some(spec) = windmill_parser::asset_parser::parse_pipeline_annotations(code).partition
-    else {
-        return Ok(None);
+    let ann = windmill_parser::asset_parser::parse_pipeline_annotations(code);
+    let in_pipeline = ann.in_pipeline;
+    let Some(spec) = ann.partition else {
+        return Ok((None, in_pipeline));
     };
 
     // Already resolved upstream — explicit run arg, backfill, or
@@ -4349,7 +4354,7 @@ async fn resolve_partition_for_job(
             .is_some_and(|s| !s.is_empty())
     });
     if already_set {
-        return Ok(None);
+        return Ok((None, in_pipeline));
     }
 
     // `dynamic` extracts from the triggering payload (the `trigger` object
@@ -4382,7 +4387,7 @@ async fn resolve_partition_for_job(
             job_id = %job.id,
             "partitioned script resolved to no partition (before start anchor); running without one"
         );
-        return Ok(None);
+        return Ok((None, in_pipeline));
     };
 
     // Persist back so dispatch_asset_triggers (which reads the producer's
@@ -4404,7 +4409,7 @@ async fn resolve_partition_for_job(
         windmill_common::worker::to_raw_value(&value),
     );
     updated.args = Some(Json(map));
-    Ok(Some(updated))
+    Ok((Some(updated), in_pipeline))
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
@@ -4566,7 +4571,8 @@ async fn handle_code_execution_job(
     // `// partitioned` (if any) and shadow `job` with a clone whose args
     // carry the resolved `partition` for the rest of execution.
     let _job_with_partition;
-    let job = match resolve_partition_for_job(job, code, conn).await? {
+    let (resolved_job, in_pipeline) = resolve_partition_for_job(job, code, conn).await?;
+    let job = match resolved_job {
         Some(j) => {
             _job_with_partition = j;
             &_job_with_partition
@@ -4619,6 +4625,7 @@ async fn handle_code_execution_job(
         lock,
         &modules,
         false,
+        in_pipeline,
     )
     .await
 }
@@ -4685,6 +4692,9 @@ pub async fn run_language_executor(
     lock: &Option<String>,
     modules: &Option<std::collections::HashMap<String, ScriptModule>>,
     run_inline: bool,
+    // Whether the script is a `// pipeline` member (parsed once upstream) — sets
+    // WM_PIPELINE so the wmll.ducklake SDK helpers record materialization state.
+    in_pipeline: bool,
 ) -> error::Result<Box<RawValue>> {
     // Defense-in-depth (GHSA-wxjq-w5pj-jqhx): the entrypoint override is
     // interpolated verbatim into a code position of the generated language
@@ -5047,6 +5057,11 @@ mount {{
 
     #[allow(unused_mut)]
     let mut envs = build_envs(envs.as_ref())?;
+    // Signal pipeline context to the script so the wmll.ducklake SDK helpers
+    // record materialization state (the grid/backfill) and skip it otherwise.
+    if in_pipeline {
+        envs.insert("WM_PIPELINE".to_string(), "true".to_string());
+    }
 
     let Some(language) = language else {
         return Err(Error::ExecutionErr(
@@ -5832,6 +5847,7 @@ pub fn init_worker_internal_server_inline_utils(
                     &None,
                     &None,
                     true,
+                    false,
                 )
                 .await
             })
@@ -5913,6 +5929,7 @@ pub fn init_worker_internal_server_inline_utils(
                     &content_info.lockfile,
                     &content_info.modules,
                     true,
+                    false,
                 )
                 .await
             })

cli/src/guidance/skills.gen.ts

@@ -1102,6 +1102,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize \`selectSql\` into a ducklake table for one
+ * partition (or the whole table when \`partition\` is omitted) — the client-side
+ * equivalent of the \`// materialize\` engine.
+ * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
 `,
   "write-script-bunnative": `---
 name: write-script-bunnative
@@ -1833,6 +1856,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize \`selectSql\` into a ducklake table for one
+ * partition (or the whole table when \`partition\` is omitted) — the client-side
+ * equivalent of the \`// materialize\` engine.
+ * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
 `,
   "write-script-csharp": `---
 name: write-script-csharp
@@ -2656,6 +2702,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize \`selectSql\` into a ducklake table for one
+ * partition (or the whole table when \`partition\` is omitted) — the client-side
+ * equivalent of the \`// materialize\` engine.
+ * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
 `,
   "write-script-duckdb": `---
 name: write-script-duckdb
@@ -4306,6 +4375,27 @@ def stream_result(stream) -> None
 #     SqlQuery instance for fetching results
 def query(sql: str, *args) -> SqlQuery
 
+# Idempotently materialize the rows of \`select_sql\` into ducklake
+# \`table\` for one \`partition\` (or the whole table when \`partition\` is
+# None). Client-side equivalent of the \`// materialize\` engine: with
+# \`unique_key\` it upserts within the slice (delete-by-key + insert);
+# without it, it replaces (whole table → CREATE OR REPLACE; partition →
+# delete the partition + insert). Re-running the same slice is safe — the
+# backfill / failure-recovery contract.
+# 
+# The partition value is bound as a DuckDB arg (never string-interpolated)
+# so it cannot inject SQL. \`select_sql\` is trusted (your own query).
+def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# INSERT-only materialization (no dedup / no replace) for an immutable
+# event-log table — for one \`partition\`, or the whole table when
+# \`partition\` is None. NOTE: unlike \`upsert_partition\`, re-running the same
+# slice duplicates rows — use only for append-only sources.
+def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# Read a materialized ducklake table, optionally a single partition.
+def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
 # Execute query and fetch results.
 # 
 # Args:

docs/ducklake-materialization.md

@@ -0,0 +1,250 @@
+# DuckLake-native materialization
+
+Design sketch for "managed, versioned, incremental" assets built on the
+DuckLake substrate. This is a companion to [`pipelines-vs-dbt.md`](./pipelines-vs-dbt.md)
+and extends its **Path C (hybrid, partition-first)** recommendation. The new
+contribution here is leveraging DuckLake's snapshot/time-travel layer, which
+the earlier doc's incremental deep-dive did not use. The annotation grammar is
+reconciled with that doc — `// partitioned` + `// unique_key` + `// append`
+stay canonical; nothing here forks a competing vocabulary.
+
+## The core reframe
+
+dbt had to *build* a materialization engine (compile SQL → `CREATE TABLE AS` /
+incremental `MERGE` / SCD2 snapshots) because the warehouse gives it nothing
+but raw SQL over a mutable table. DuckLake hands us, at the storage layer, the
+four things that engine exists to provide:
+
+| Capability | Source |
+|---|---|
+| ACID multi-statement transactions | DuckLake |
+| A snapshot per commit + time-travel (`AT (VERSION => n)` / `AT (TIMESTAMP => ...)`) | DuckLake |
+| Physical partitioning + pruning (`ALTER TABLE … SET PARTITIONED BY (…)`) | DuckLake |
+| Schema evolution tracked in the catalog DB | DuckLake |
+
+Windmill already attaches DuckLake fully — see `transform_attach_ducklake`
+(`backend/windmill-worker/src/duckdb_executor.rs:661`), which rewrites a user's
+`ATTACH 'ducklake://name' AS dl` into the real
+`ATTACH 'ducklake:postgres:…' AS dl (DATA_PATH 's3://…', OVERRIDE_DATA_PATH
+TRUE, AUTOMATIC_MIGRATION TRUE)` at `duckdb_executor.rs:730`. But today every
+write is a destructive overwrite and **all four capabilities above are thrown
+away** — snapshots are never surfaced, partitioning is purely an orchestration
+concept disconnected from the physical layout.
+
+So the materialization engine we need is a thin layer — a write-strategy
+wrapper + snapshot capture — not a dbt rebuild. This is exactly the
+"buy A's 80% without B's dialect-rewriting tax" tradeoff `pipelines-vs-dbt.md`
+argued for; DuckLake is what makes the remaining 20% (versioning, reproducible
+reads, materialization history) nearly free instead of a second project.
+
+## Annotation grammar (final)
+
+One self-documenting line; managed-by-default. Strategy options live *on* the
+`materialize` line (they have no meaning without it), while `// partitioned`
+stays separate because it is cross-cutting (cascade + scheduling + materialize).
+
+```
+// materialize ducklake://analytics/orders_daily              → managed, replace (default)
+// materialize ducklake://analytics/orders_daily key=order_id → managed, merge
+// materialize ducklake://analytics/orders_daily append       → managed, append
+// materialize manual ducklake://analytics/orders_daily       → track-only escape hatch
+```
+
+- **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill
+  generates the write DDL, captures the DuckLake snapshot, and records state.
+  DuckDB-only; validated at deploy (a non-SELECT script is rejected with a clear
+  error pointing to the `wmll.ducklake` helpers).
+- **`manual`** — escape hatch: the script writes its own DDL; Windmill only
+  records state (no snapshot capture, no idempotency guarantee). Rare; explicit.
+- **`key=<col>`** → MERGE (dedup within slice); **`append`** → INSERT-only;
+  neither → DELETE-by-partition + INSERT (replace). `append` wins over `key` if
+  both are given (deploy warning).
+- **`// partitioned <kind>`** — unit of work + state + backfill (separate;
+  cross-cutting). Polyglot / multi-statement writes use the `wmll.ducklake`
+  helpers instead of `// materialize`.
+
+There is no `wrap` keyword — `materialize` *is* "manage the write," so it was
+redundant; the only reason for it was to carve out the weak track-only mode,
+which is now the explicit `manual` opt-out.
+
+DuckLake snapshots are **orthogonal to all of the above** — they apply to every
+strategy automatically because every write is a DuckLake commit. The user never
+annotates for versioning; they get it.
+
+## Executor codegen
+
+### The seam
+
+`run_duckdb` already splits the script into statement blocks and rewrites
+custom `ATTACH` blocks in a single pass before execution
+(`duckdb_executor.rs:114-160`):
+
+```rust
+let query_block_list = parse_sql_blocks(&query, true);
+// each block: remove_comments → if ducklake/datatable ATTACH, expand; else passthrough
+```
+
+All blocks run in order on one DuckDB connection. Materialize has two modes:
+
+1. **Managed (default).** The user writes *setup + one trailing `SELECT`*. Windmill
+   replaces that SELECT with generated statements, wrapped in an *explicit
+   DuckLake transaction it controls* — never textual `BEGIN/COMMIT` injected
+   around the user's other statements (fragile across their own `ATTACH`s and
+   multi-statement SQL). For a partitioned `replace` the SELECT block expands to:
+
+   ```sql
+   -- generated for: // partitioned daily ; target = dl.orders_daily ; partition = '2026-06-19'
+   CREATE TABLE IF NOT EXISTS dl.orders_daily AS
+     SELECT *, CAST(NULL AS VARCHAR) AS _wm_partition FROM (<user_select>) WHERE false;  -- first-run bootstrap
+   ALTER TABLE dl.orders_daily SET PARTITIONED BY (_wm_partition);
+   BEGIN TRANSACTION;
+   DELETE FROM dl.orders_daily WHERE _wm_partition = '2026-06-19';
+   INSERT INTO dl.orders_daily SELECT *, '2026-06-19' AS _wm_partition FROM (<user_select>);
+   COMMIT;
+   ```
+
+   The strategy variants are all DELETE+INSERT-shaped — no `MERGE INTO`, which
+   DuckLake can't reliably run on a fresh partition (it 404s writing the first
+   rows):
+   - **whole-table replace** (no `// partitioned`) → a single `CREATE OR REPLACE
+     TABLE … AS <user_select>` (handles schema changes, still snapshots).
+   - **`key=<col>`** → `DELETE FROM … WHERE [<partition> AND] <col> IN (SELECT
+     <col> FROM (<user_select>))` then `INSERT` (upsert within the slice).
+   - **`append`** → the `DELETE` is dropped (insert-only).
+
+2. **`manual`.** The user writes their own DDL inside their own `BEGIN … COMMIT`;
+   Windmill injects nothing into the body and only records state (no snapshot
+   capture, no idempotency guarantee).
+
+The `_wm_partition` column is the physical link the orchestration layer lacks: on
+first materialize Windmill runs `ALTER TABLE … SET PARTITIONED BY (_wm_partition)`
+so DuckLake prunes on read and DELETE-by-partition rewrites only that partition's
+Parquet files.
+
+> Storage: DuckLake writes go to `s3://_default_/` through the windmill S3 proxy
+> (`/api/w/{ws}/s3_proxy`, gated behind the `parquet` + `private` features). The
+> proxy must sign the SigV4 canonical URI with **single** percent-encoding — the
+> SigV4 default (`Double`) 401s Hive-partition keys like `_wm_partition=2026-06-19`
+> (the `=` double-encodes to `%253D` vs the client's `%3D`).
+
+### Run summary capture
+
+After the generated blocks, Windmill appends one read block — it is both the job's
+result (a useful preview rendered as the materialized table) and the row it records:
+
+```sql
+SELECT 'ducklake://<name>/<table>' AS materialized,
+       '<partition>' AS partition,            -- only when partitioned
+       (SELECT count(*) FROM <target> [WHERE _wm_partition = '<partition>']) AS rows,
+       (SELECT max(snapshot_id) FROM ducklake_snapshots('<target>')) AS snapshot_id;
+```
+
+The `snapshot_id` and `rows` are persisted as `materialized_partition` metadata.
+One extra round-trip per materialization, no new infra.
+
+## Metadata schema
+
+Extends the `materialized_partitions` table proposed in `pipelines-vs-dbt.md`
+§"First implementation slice" with the DuckLake snapshot id:
+
+```
+materialized_partition (
+  workspace_id      TEXT,
+  asset_kind        TEXT,        -- 'ducklake'
+  asset_path        TEXT,        -- 'analytics/orders_daily'
+  partition         TEXT,        -- '2026-06-19'  (NULL for unpartitioned)
+  snapshot_id       BIGINT,      -- DuckLake snapshot produced by this materialize
+  row_count         BIGINT,
+  job_id            UUID,
+  materialized_at   TIMESTAMPTZ,
+  PRIMARY KEY (workspace_id, asset_kind, asset_path, partition)
+)
+```
+
+This one table drives four things at once:
+
+- **Observability** — "last materialized: snapshot 42, 1.2M rows, 09:14" per
+  asset node (closes the Dagster-catalog gap from the v1-readiness review).
+- **Run-stale / gap detection** — which partitions exist, which are missing.
+- **Backfill** — the missing/failed set *is* the backfill worklist.
+- **Snapshot pinning** — see below.
+
+## Reproducibility — the beyond-dbt part
+
+Because every materialization records the snapshot it produced, a downstream
+consumer can read the *exact* upstream snapshot its run saw:
+
+```sql
+FROM dl.orders_daily AT (VERSION => $WM_UPSTREAM_SNAPSHOT)
+```
+
+The cascade already threads a `trigger` blob (producer path, partition) to each
+subscriber; add the producer's captured `snapshot_id` to it, and a consumer's
+read is pinned to the upstream state at dispatch time. That makes the *whole
+pipeline* reproducible and time-travelable — something dbt has no native answer
+for (dbt models are always "whatever's in the warehouse now"). It also gives
+rollback (re-point an asset to snapshot N) and "what did this table look like at
+the failing run" debugging, for free off the same captured ids.
+
+This is the differentiator worth leaning on. It is not catch-up to dbt; it is a
+capability dbt structurally cannot offer, and DuckLake gives it to us at the
+cost of recording one integer per run.
+
+It also means **we do not build SCD2 snapshots** (gap #4 in `pipelines-vs-dbt.md`):
+DuckLake time-travel is a strictly better answer for most of what dbt's
+`{% snapshot %}` is used for. One fewer engine to write.
+
+## Scoping decision: DuckLake vs DataTable
+
+**Make DuckLake the materialization/versioning substrate; keep DataTable as the
+live operational table with no versioning.** DataTable is plain Postgres
+(`transform_attach_datatable`, `duckdb_executor.rs:742`) — no native snapshots
+or time-travel — so giving *it* the versioned/incremental story means building
+MVCC-on-top ourselves (history tables, SCD2), precisely the complexity this
+DuckLake approach exists to avoid. Clean split:
+
+- `ducklake://` → analytics, versioned, reproducible, backfillable.
+- `datatable://` → mutable app/operational state; partition idempotency via
+  DELETE+INSERT still works, but no snapshot/time-travel layer.
+
+Don't try to give both the full treatment for v1.
+
+## v1 slice (smallest viable)
+
+1. **Partition runtime context** — resolve `(value, start, end)` and surface as
+   `WM_PARTITION*` bind/env (Path C step 1; partly built per the
+   pipeline-partition-runtime work).
+2. **Physical partition wiring** — `_wm_partition` column + `SET PARTITIONED BY`
+   on first materialize for `ducklake://` targets.
+3. **Strategy templates** — DELETE+INSERT default (`CREATE OR REPLACE` for the
+   whole table); delete-by-key + insert when `key=<col>`; INSERT-only when
+   `append`. Managed `// materialize` wraps a single-SELECT DuckDB script behind
+   these templates; `// materialize manual` opts out.
+4. **Snapshot + metadata capture** — append `ducklake_snapshots` read, persist
+   `materialized_partition` rows.
+5. **Surface it** — last-materialized/snapshot/row-count on the asset node;
+   missing-partition set feeds the backfill UI.
+6. *v1.x* — snapshot pinning across the cascade (`$WM_UPSTREAM_SNAPSHOT`),
+   rollback, time-travel read helper.
+
+Steps 1–5 are a thin annotation+template layer plus one metadata table and one
+extra read per run. They deliver managed/incremental/versioned assets,
+idempotent partitioned materialization, the backfill substrate, and
+materialization observability together — and stay recognizably Windmill-shaped.
+
+## Open decisions
+
+These ride on top of the six in `pipelines-vs-dbt.md` §"Decisions either path
+forces"; DuckLake-specific:
+
+1. **Bootstrap of `SET PARTITIONED BY`.** First-materialize detection — table
+   absent vs. present-but-unpartitioned. Idempotent re-apply.
+2. **Snapshot retention / compaction.** DuckLake snapshots accumulate; when do
+   we expire old ones, and does pinning hold a snapshot alive past retention?
+3. **Pin scope.** Pin only direct producers, or the full transitive upstream
+   set per run? Storage and "stale pin" semantics differ.
+4. **Managed multi-statement.** *Resolved:* managed `// materialize` accepts
+   setup statements (ATTACH/SET/…) followed by exactly one trailing SELECT, and
+   rejects anything else at deploy with a clear error pointing to
+   `// materialize manual`. The classifier (`sql_materialize.rs`) is the single
+   source of truth.

docs/pipelines-vs-dbt.md

@@ -280,6 +280,11 @@ introspection per substrate is its own project.
 Ships A's 80% case first without committing to B's dialect-rewriting tax.
 Wrapping becomes opt-in convenience for users who want dbt-style ergonomics.
 
+> See [`ducklake-materialization.md`](./ducklake-materialization.md) for the
+> DuckLake-native realization of this path: how snapshots make the assets
+> versioned/reproducible for free, the executor codegen seam, and the
+> materialization-metadata schema.
+
 ### Decisions either path forces
 
 1. **Partition window provenance.** Scheduler tick? Trigger event time

frontend/src/lib/components/DisplayResult.svelte

@@ -7,7 +7,7 @@
 	import { base } from '$lib/base'
 	import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile'
 	import { appendViewToken } from '$lib/viewToken'
-	import { Button, Drawer, DrawerContent } from './common'
+	import { Badge, Button, Drawer, DrawerContent } from './common'
 	import {
 		ClipboardCopy,
 		Download,
@@ -16,8 +16,10 @@
 		Braces,
 		Highlighter,
 		ArrowDownFromLine,
+		Database,
 		Loader2
 	} from 'lucide-svelte'
+	import DucklakeResultPreview from './assets/AssetGraph/DucklakeResultPreview.svelte'
 	import Portal from '$lib/components/Portal.svelte'
 	import DisplayResultControlBar from './DisplayResultControlBar.svelte'
 
@@ -68,6 +70,7 @@
 		| 'filename'
 		| 's3object'
 		| 's3object-list'
+		| 'materialized'
 		| 'plain'
 		| 'markdown'
 		| 'map'
@@ -195,6 +198,28 @@
 		return keys.includes('s3') && typeof result.s3 === 'string'
 	}
 
+	// The materialize-run summary — `[{ materialized: 'ducklake://…', rows,
+	// snapshot_id }]` or the bare object. The shape is narrow (a `ducklake://`
+	// value plus a `snapshot_id` key) so an ordinary user result isn't hijacked.
+	function parseMaterializedResult(
+		res: any
+	):
+		| { materialized: string; partition?: string; rows?: number; snapshot_id?: number | null }
+		| undefined {
+		const obj = Array.isArray(res) && res.length === 1 ? res[0] : res
+		if (
+			obj &&
+			typeof obj === 'object' &&
+			typeof obj.materialized === 'string' &&
+			obj.materialized.startsWith('ducklake://') &&
+			'snapshot_id' in obj
+		) {
+			return obj
+		}
+		return undefined
+	}
+
+	let showMaterializedPreview = $state(true)
 	let is_render_all = $state(false)
 	let download_as_csv = $state(false)
 	function inferResultKind(result: any) {
@@ -221,6 +246,13 @@
 			}
 			try {
 				let keys = result && typeof result === 'object' ? Object.keys(result) : []
+
+				if (parseMaterializedResult(result)) {
+					largeObject = false
+					is_render_all = false
+					return 'materialized'
+				}
+
 				is_render_all =
 					keys.length == 1 && keys.includes('render_all') && Array.isArray(result['render_all'])
 
@@ -832,6 +864,43 @@
 							></div
 						>
 					</div>
+				{:else if !forceJson && resultKind === 'materialized'}
+					{@const m = parseMaterializedResult(result)}
+					{#if m}
+						<div class="flex flex-col gap-2 w-full">
+							<div class="flex items-center gap-2 flex-wrap text-xs">
+								<Database size={14} class="text-tertiary shrink-0" />
+								<span class="font-mono text-emphasis break-all">{m.materialized}</span>
+								{#if m.partition}
+									<Badge color="blue">partition {m.partition}</Badge>
+								{/if}
+								{#if typeof m.rows === 'number'}
+									<Badge color="green">
+										{m.rows}
+										{m.rows === 1 ? 'row' : 'rows'}{m.partition ? ' in partition' : ''}
+									</Badge>
+								{/if}
+								{#if m.snapshot_id != null}
+									<Badge color="gray">snapshot {m.snapshot_id}</Badge>
+								{/if}
+							</div>
+							<Toggle
+								class="flex"
+								bind:checked={showMaterializedPreview}
+								size="xs"
+								options={{ right: 'Preview rows' }}
+							/>
+							{#if showMaterializedPreview}
+								<div class="border rounded-md h-80 min-h-0 overflow-hidden">
+									<DucklakeResultPreview
+										assetUri={m.materialized}
+										partition={m.partition}
+										class="h-full"
+									/>
+								</div>
+							{/if}
+						</div>
+					{/if}
 				{:else if !forceJson && resultKind === 's3object'}
 					{@const s3object = parseS3Object(result) as typeof result}
 					<div

frontend/src/lib/components/ScriptEditor.svelte

@@ -15,6 +15,7 @@
 	import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils'
 	import Editor from './Editor.svelte'
 	import { inferArgs, inferAssets, inferAnsibleExecutionMode } from '$lib/infer'
+	import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations'
 	import { isWorkflowAsCode } from '$lib/components/graph/wacToFlow'
 	import WacDiagram from '$lib/components/graph/WacDiagram.svelte'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
@@ -878,16 +879,67 @@
 				// we reapply initial args as the schema form might have cleared them between mount and the schema inference
 				args = initialArgs
 			}
+			injectPartitionArg(nschema, args, nlang ?? lang, code)
 			schema = nschema
 		} catch (e) {
 			validCode = false
 		}
 	}
 
+	// A `// partitioned` pipeline script is materialized one slice at a time and
+	// receives the slice as a runtime `partition` arg (the cascade injects it in
+	// production). It isn't a code parameter, so schema inference doesn't see it —
+	// surface it in the test form so a partitioned script can be run manually.
+	function injectPartitionArg(
+		s: any,
+		a: Record<string, any> | undefined,
+		l: string | undefined,
+		c: string
+	) {
+		try {
+			if (l !== 'duckdb' || !s?.properties) return
+			const part = parsePipelineAnnotations(c).partition
+			if (!part) return
+			// Date-based partition kinds render a date / datetime picker; a dynamic
+			// key is a free-form string.
+			const format =
+				part.kind === 'hourly'
+					? 'date-time'
+					: part.kind === 'daily' || part.kind === 'weekly' || part.kind === 'monthly'
+						? 'date'
+						: undefined
+			if (!s.properties['partition']) {
+				s.properties['partition'] = {
+					type: 'string',
+					...(format ? { format } : {}),
+					// ISO output so partition keys sort lexicographically (the date
+					// picker defaults to dd-MM-yyyy otherwise).
+					...(format === 'date' ? { dateFormat: 'yyyy-MM-dd' } : {}),
+					description:
+						part.kind === 'dynamic'
+							? 'Partition key value to materialize.'
+							: `Partition (${part.kind}) to materialize.`
+				}
+				if (Array.isArray(s.order) && !s.order.includes('partition')) {
+					s.order = ['partition', ...s.order]
+				}
+			}
+			// Pre-fill the *test* arg with the current slice for date kinds — a
+			// convenience default, kept on the args (not baked into the schema,
+			// where it would persist to the deployed script and go stale).
+			if (format && a && (a['partition'] == null || a['partition'] === '')) {
+				const now = new Date()
+				a['partition'] =
+					format === 'date' ? now.toISOString().slice(0, 10) : now.toISOString().slice(0, 16)
+			}
+		} catch (e) {}
+	}
+
 	async function inferModuleSchema() {
 		if (activeModuleTab === null) return
 		try {
 			await inferArgs(effectiveLang, editorCode, testPanelSchema)
+			injectPartitionArg(testPanelSchema, testPanelArgs, effectiveLang, editorCode)
 			moduleTestState[activeModuleTab] = { args: testPanelArgs, schema: testPanelSchema }
 		} catch (e) {
 			// Module code may be in-progress; silently ignore

frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte

@@ -29,6 +29,7 @@
 	import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte'
 	import S3FilePreview from '$lib/components/S3FilePreview.svelte'
 	import DataTablePreview from './DataTablePreview.svelte'
+	import PartitionStatusGrid from './PartitionStatusGrid.svelte'
 	import AssetRunsPanel from './AssetRunsPanel.svelte'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
 	import { fade } from 'svelte/transition'
@@ -571,6 +572,12 @@
 					// deployed script, or a draft promoted from unsaved edits
 					// to a deployed script) chains off it.
 					parent_hash: script.hash ? String(script.hash) : undefined,
+					// Let the backend resolve the parent to the current head for
+					// this path (atomically, under an advisory lock) instead of
+					// rejecting a stale parent_hash with a "lineage must be
+					// linear" error — the pane is opened from a graph snapshot
+					// that can fall behind the deployed head between renders.
+					auto_parent: true,
 					is_template: false,
 					tag: script.tag,
 					kind: script.kind as Script['kind'] | undefined,
@@ -976,6 +983,8 @@
 									class="h-full"
 									refreshKey={previewRefreshKey}
 								/>
+							{:else if selection.asset_kind === 'ducklake'}
+								<PartitionStatusGrid path={selection.path} {workspace} />
 							{:else}
 								<div class="p-3 text-xs text-secondary">
 									No inline preview yet for {selection.asset_kind}. Use the producer/consumer arrows

frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte

@@ -0,0 +1,79 @@
+<script lang="ts">
+	import { Button } from '$lib/components/common'
+	import Modal from '$lib/components/common/modal/Modal.svelte'
+	import DateInput from '$lib/components/DateInput.svelte'
+	import { enterpriseLicense } from '$lib/stores'
+
+	// Backfill re-runs the CE materialization once per partition in [from, to].
+	// It is an enterprise feature (orchestration over a range); the dialog is
+	// only reachable when licensed, but we guard here too so the action can
+	// never fire in CE.
+	interface Props {
+		// Controlled by the parent. `$bindable()` without a default per the
+		// AGENTS.md ban on `$bindable(default)` for optional props.
+		open?: boolean
+		assetPath: string
+		// Invoked with the inclusive ISO date range; the parent performs the
+		// actual fan-out (EE backfill endpoint).
+		onBackfill: (from: string, to: string) => Promise<void>
+	}
+
+	let { open = $bindable(), assetPath, onBackfill }: Props = $props()
+
+	let fromDate = $state<string | undefined>(undefined)
+	let toDate = $state<string | undefined>(undefined)
+	let loading = $state(false)
+	let error = $state<string | undefined>(undefined)
+
+	let canSubmit = $derived(!!$enterpriseLicense && !!fromDate && !!toDate && !loading)
+
+	async function submit() {
+		if (!fromDate || !toDate) return
+		loading = true
+		error = undefined
+		try {
+			await onBackfill(fromDate, toDate)
+			open = false
+		} catch (e) {
+			error = e instanceof Error ? e.message : String(e)
+		} finally {
+			loading = false
+		}
+	}
+</script>
+
+<Modal bind:open={() => open ?? false, (v) => (open = v)} title={`Backfill ${assetPath}`}>
+	<div class="flex flex-col gap-4">
+		{#if !$enterpriseLicense}
+			<p class="text-sm text-secondary">
+				Partition backfill is an enterprise feature. Materializing a single partition is available
+				in the open-source edition; reprocessing a historical range requires an enterprise license.
+			</p>
+		{:else}
+			<p class="text-sm text-secondary">
+				Re-runs the materialization for each partition in the range. Re-running a partition is
+				idempotent, so this is safe to repeat.
+			</p>
+			<div class="flex gap-3">
+				<div class="flex flex-col gap-1">
+					<span class="text-xs font-semibold">From</span>
+					<DateInput bind:value={fromDate} />
+				</div>
+				<div class="flex flex-col gap-1">
+					<span class="text-xs font-semibold">To</span>
+					<DateInput bind:value={toDate} />
+				</div>
+			</div>
+			{#if error}
+				<p class="text-sm text-red-600">{error}</p>
+			{/if}
+		{/if}
+	</div>
+
+	{#snippet actions()}
+		<Button variant="subtle" onclick={() => (open = false)}>Cancel</Button>
+		<Button variant="accent" disabled={!canSubmit} {loading} onclick={submit}>
+			Start backfill
+		</Button>
+	{/snippet}
+</Modal>

frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte

@@ -0,0 +1,136 @@
+<script lang="ts">
+	// Live row preview for a ducklake table, used inside the materialize result
+	// display (DisplayResult). Loads the table's column metadata then renders the
+	// shared read-only DBTable grid (paged). Mirrors DataTablePreview but for the
+	// `ducklake` input type — a ducklake catalog is always attachable, so there
+	// is no "configure connection" branch.
+	//
+	// For a partitioned target the preview can scope to the just-written slice
+	// ("This partition") or show the full table ("Whole table") via a toggle.
+	import DBTable from '$lib/components/DBTable.svelte'
+	import { resource } from 'runed'
+	import { workspaceStore } from '$lib/stores'
+	import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata'
+	import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps'
+	import type { DbInput } from '$lib/components/dbTypes'
+	import { parseDbInputFromAssetSyntax } from '$lib/utils'
+	import { AlertTriangle, Loader2 } from 'lucide-svelte'
+	import { twMerge } from 'tailwind-merge'
+	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
+	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
+
+	interface Props {
+		// Full asset URI, e.g. `ducklake://main/orders_daily`.
+		assetUri: string
+		// When set the target is partitioned and the preview can scope to this slice.
+		partition?: string
+		// Bump to force a re-fetch (parallel to DataTablePreview's refreshKey).
+		refreshKey?: any
+		class?: string
+	}
+	let { assetUri, partition, refreshKey, class: className = '' }: Props = $props()
+
+	// Default to the just-written slice; `scoped` is a no-op without a partition
+	// (the toggle is only shown when there is one).
+	let scope = $state<'partition' | 'whole'>('partition')
+	let scoped = $derived(!!partition && scope === 'partition')
+
+	let input = $derived<DbInput | undefined>(parseDbInputFromAssetSyntax(assetUri) ?? undefined)
+	let table = $derived(
+		input && 'specificTable' in input ? (input.specificTable as string | undefined) : undefined
+	)
+	// Explicit schema from `ducklake://<lake>/<schema>.<table>` (default `main`).
+	// The preview SELECT is `FROM <tableKey>` unquoted, so a `schema.table` key
+	// resolves to the right schema — without this the read hits the default one.
+	let schema = $derived(
+		input && 'specificSchema' in input ? (input.specificSchema as string | undefined) : undefined
+	)
+	let tableKey = $derived(schema && table ? `${schema}.${table}` : table)
+
+	// Scope rows to the partition slice via the preview SELECT's whereClause.
+	// The value is single-quote-escaped (it's raw-injected server-side).
+	let whereClause = $derived(
+		scoped && partition ? `_wm_partition = '${partition.replaceAll("'", "''")}'` : undefined
+	)
+
+	let colDefs = resource(
+		() => [input, refreshKey] as const,
+		async ([_input]) => {
+			if (!_input || !$workspaceStore) return undefined
+			try {
+				return await loadAllTablesMetaData($workspaceStore, _input)
+			} catch {
+				// A load failure reads the same as "table missing" from the preview's
+				// POV; the underlying error is surfaced by loadAllTablesMetaData.
+				return undefined
+			}
+		}
+	)
+
+	let tableColDefs = $derived.by(() => {
+		const defs = colDefs.current
+		if (!table || !defs) return undefined
+		const direct = defs[`${schema ?? 'main'}.${table}`] ?? defs[table]
+		const found =
+			direct ??
+			(() => {
+				const key = Object.keys(defs).find((k) => k === table || k.endsWith(`.${table}`))
+				return key ? defs[key] : undefined
+			})()
+		if (!found) return undefined
+		// Hide the internal partition column when scoped to one partition (every
+		// row has the same value); keep it in whole-table view so the rows from
+		// different partitions are distinguishable.
+		return scoped ? found.filter((c: { field?: string }) => c.field !== '_wm_partition') : found
+	})
+
+	let dbTableOps = $derived.by(() => {
+		if (!(input && tableColDefs && tableKey && $workspaceStore)) return undefined
+		const ops = dbTableOpsWithPreviewScripts({
+			input,
+			tableKey,
+			colDefs: tableColDefs,
+			workspace: $workspaceStore,
+			whereClause
+		})
+		// Read-only preview: drop the mutation handlers so DBTable hides its
+		// edit / delete / insert affordances — this is a result view, not a
+		// table editor (and the table is overwritten on the next materialize).
+		const readOnly = { ...ops }
+		delete readOnly.onUpdate
+		delete readOnly.onDelete
+		delete readOnly.onInsert
+		return readOnly
+	})
+</script>
+
+<div class={twMerge('flex flex-col min-h-0 relative', className)}>
+	{#if partition}
+		<div class="pb-2">
+			<ToggleButtonGroup selected={scope} on:selected={(e) => (scope = e.detail)}>
+				{#snippet children({ item })}
+					<ToggleButton size="sm" value="partition" label="This partition" {item} />
+					<ToggleButton size="sm" value="whole" label="Whole table" {item} />
+				{/snippet}
+			</ToggleButtonGroup>
+		</div>
+	{/if}
+	{#if colDefs.loading && !colDefs.current}
+		<div class="flex items-center justify-center p-4 text-tertiary">
+			<Loader2 class="animate-spin" size={18} />
+		</div>
+	{:else if !tableColDefs}
+		<div class="flex items-center gap-2 p-3 text-2xs text-tertiary">
+			<AlertTriangle size={14} class="text-amber-500" />
+			Couldn't load a preview of this table.
+		</div>
+	{:else if dbTableOps}
+		<!-- Re-mount when the scope toggles so DBTable re-fetches with the new
+		     whereClause / column set. -->
+		{#key [refreshKey, scope]}
+			<div class="grow min-h-0">
+				<DBTable {dbTableOps} />
+			</div>
+		{/key}
+	{/if}
+</div>

frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte

@@ -0,0 +1,134 @@
+<script lang="ts">
+	import { resource } from 'runed'
+	import { OpenAPI } from '$lib/gen'
+	import { Button } from '$lib/components/common'
+	import { Loader2, RefreshCw, History } from 'lucide-svelte'
+	import { sendUserToast } from '$lib/utils'
+	import BackfillRangeDialog from './BackfillRangeDialog.svelte'
+
+	interface Props {
+		// The materialized ducklake asset path (`<ducklake>/<table>`).
+		path: string
+		workspace: string
+	}
+	let { path, workspace }: Props = $props()
+
+	type MaterializedPartition = {
+		partition: string
+		status: 'running' | 'materialized' | 'failed'
+		snapshot_id?: number | null
+		row_count?: number | null
+		materialized_at: string
+		error?: string | null
+	}
+
+	let partitions = resource([() => workspace, () => path], async ([ws, p], _prev, { signal }) => {
+		if (!ws || !p) return [] as MaterializedPartition[]
+		const res = await fetch(
+			`${OpenAPI.BASE ?? ''}/w/${ws}/assets/partitions?path=${encodeURIComponent(p)}`,
+			{ credentials: 'include', signal }
+		)
+		if (!res.ok) throw new Error(`GET /assets/partitions → ${res.status}`)
+		return (await res.json()) as MaterializedPartition[]
+	})
+
+	let backfillOpen = $state(false)
+
+	const statusClass: Record<MaterializedPartition['status'], string> = {
+		materialized: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300',
+		running: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300',
+		failed: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300'
+	}
+
+	async function onBackfill(from: string, to: string) {
+		// The fan-out runner is an enterprise feature; the dialog only submits
+		// when licensed. This posts the intent to the (EE) backfill endpoint.
+		const res = await fetch(`${OpenAPI.BASE ?? ''}/w/${workspace}/assets/backfill`, {
+			method: 'POST',
+			credentials: 'include',
+			headers: { 'content-type': 'application/json' },
+			body: JSON.stringify({ path, from, to })
+		})
+		if (!res.ok) throw new Error(`backfill → ${res.status}`)
+		sendUserToast(`Backfill queued for ${path} (${from} → ${to})`)
+		await partitions.refetch()
+	}
+</script>
+
+<div class="flex flex-col h-full">
+	<div class="flex items-center justify-between gap-2 px-3 py-2 border-b shrink-0">
+		<span class="text-xs font-semibold text-secondary">Materialized partitions</span>
+		<div class="flex items-center gap-1">
+			<Button
+				variant="subtle"
+				unifiedSize="sm"
+				startIcon={{ icon: RefreshCw }}
+				iconOnly
+				onclick={() => partitions.refetch()}
+				title="Refresh"
+			/>
+			<!-- The backfill range runner (POST /assets/backfill) is a planned
+			     enterprise follow-up and not yet implemented on the backend, so the
+			     button stays disabled — enabling it would 404. Re-enable when the
+			     endpoint lands. -->
+			<Button
+				variant="default"
+				unifiedSize="sm"
+				startIcon={{ icon: History }}
+				disabled
+				onclick={() => (backfillOpen = true)}
+				title="Backfill a range of partitions — coming soon (enterprise)"
+			>
+				Backfill
+			</Button>
+		</div>
+	</div>
+
+	<div class="flex-1 min-h-0 overflow-auto p-3">
+		{#if partitions.loading}
+			<div class="flex items-center gap-2 text-tertiary text-xs">
+				<Loader2 size={14} class="animate-spin" /> Loading partitions…
+			</div>
+		{:else if partitions.error}
+			<p class="text-xs text-red-600">Failed to load: {partitions.error.message}</p>
+		{:else if !partitions.current?.length}
+			<p class="text-xs text-secondary">
+				No partitions materialized yet. They appear here after a <span class="font-mono"
+					>// materialize</span
+				> run.
+			</p>
+		{:else}
+			<table class="w-full text-xs">
+				<thead class="text-tertiary text-left">
+					<tr>
+						<th class="font-medium pb-1 pr-2">Partition</th>
+						<th class="font-medium pb-1 pr-2">Status</th>
+						<th class="font-medium pb-1 pr-2">Snapshot</th>
+						<th class="font-medium pb-1 pr-2">Rows</th>
+						<th class="font-medium pb-1">Materialized</th>
+					</tr>
+				</thead>
+				<tbody>
+					{#each partitions.current as p (p.partition)}
+						<tr class="border-t">
+							<td class="py-1 pr-2 font-mono">{p.partition || '(whole table)'}</td>
+							<td class="py-1 pr-2">
+								<span class="px-1.5 py-0.5 rounded text-3xs font-medium {statusClass[p.status]}">
+									{p.status}
+								</span>
+							</td>
+							<td class="py-1 pr-2 font-mono">{p.snapshot_id ?? '—'}</td>
+							<td class="py-1 pr-2">{p.row_count ?? '—'}</td>
+							<td class="py-1 text-tertiary">{new Date(p.materialized_at).toLocaleString()}</td>
+						</tr>
+						{#if p.error}
+							<tr><td colspan="5" class="pb-1 text-3xs text-red-600 font-mono">{p.error}</td></tr>
+						{/if}
+					{/each}
+				</tbody>
+			</table>
+		{/if}
+	</div>
+</div>
+
+<BackfillRangeDialog bind:open={backfillOpen} assetPath={path} {onBackfill} />

frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte

@@ -231,7 +231,7 @@
 	<div
 		bind:this={outputEl}
 		class={twMerge(
-			'flex flex-col gap-1 p-2 grow w-56 overflow-auto transition-opacity',
+			'flex flex-col gap-1 p-2 grow w-80 overflow-auto transition-opacity',
 			selected.triggerId && selected.language ? '' : 'opacity-20'
 		)}
 		{@attach arrowTabNav({ onKeyDown: selectAndAdvanceTo(() => pathEl, { timeout: 50 }) })}
@@ -240,7 +240,7 @@
 		{#each visibleOutputKinds.length ? visibleOutputKinds : PIPELINE_OUTPUT_KINDS as k}
 			{@const isSelected = selected.outputId === k.id}
 			<Button variant="subtle" selected={isSelected} onClick={() => (selected.outputId = k.id)}>
-				<span class="flex flex-col items-start flex-1 min-w-0">
+				<span class="flex flex-col items-start flex-1 min-w-0 text-left">
 					<span class="text-xs font-normal leading-tight">{k.label}</span>
 					{#if k.description}
 						<span

frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts

@@ -17,7 +17,8 @@ const ASSERTED_TS_FIELDS: Record<keyof PipelineAnnotations, true> = {
 	partition: true,
 	freshness: true,
 	tag: true,
-	retry: true
+	retry: true,
+	materialize: true
 }
 
 // Parser-parity guard: this TS parser (drives the live graph preview) and
@@ -58,6 +59,13 @@ type Fixture = {
 		freshness: string | null
 		tag: string | null
 		retry: { count: number; delay: string | null } | null
+		materialize?: {
+			target_kind: string
+			target_path: string
+			manual?: boolean
+			append?: boolean
+			unique_key?: string | null
+		} | null
 	}
 }
 
@@ -126,6 +134,26 @@ describe('parsePipelineAnnotations matches the shared Rust fixture corpus', () =
 				expect(got.retry?.count, 'retry count').toBe(f.expected.retry.count)
 				expect(got.retry?.delay, 'retry delay').toEqual(f.expected.retry.delay ?? undefined)
 			}
+
+			if (f.expected.materialize == null) {
+				expect(got.materialize, 'materialize').toBeUndefined()
+			} else {
+				expect(got.materialize?.targetKind, 'materialize target kind').toBe(
+					f.expected.materialize.target_kind
+				)
+				expect(got.materialize?.targetPath, 'materialize target path').toBe(
+					f.expected.materialize.target_path
+				)
+				expect(got.materialize?.manual ?? false, 'materialize manual').toBe(
+					f.expected.materialize.manual ?? false
+				)
+				expect(got.materialize?.append ?? false, 'materialize append').toBe(
+					f.expected.materialize.append ?? false
+				)
+				expect(got.materialize?.uniqueKey, 'materialize key').toEqual(
+					f.expected.materialize.unique_key ?? undefined
+				)
+			}
 		})
 	}
 })

frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts

@@ -74,6 +74,21 @@ export type RetrySpec = {
 	delay?: string
 }
 
+// `// materialize [manual] <asset> [append] [key=<col>]` — see backend
+// MaterializeSpec. Managed by default (the runtime generates the write DDL
+// around a single SELECT); `manual` opts out (the script writes its own DDL,
+// track-only). `append` / `key` are managed-mode strategy options.
+export type MaterializeSpec = {
+	targetKind: AssetKind
+	targetPath: string
+	// track-only escape hatch; absent === false (managed)
+	manual?: boolean
+	// INSERT-only strategy; absent === false
+	append?: boolean
+	// merge key; absent === replace (or append)
+	uniqueKey?: string
+}
+
 export type PipelineAnnotations = {
 	inPipeline: boolean
 	triggerAssets: PipelineTriggerAsset[]
@@ -82,6 +97,8 @@ export type PipelineAnnotations = {
 	freshness?: FreshnessSpec
 	tag?: string
 	retry?: RetrySpec
+	// `// materialize [manual] <asset> [append] [key=<col>]` — target + strategy.
+	materialize?: MaterializeSpec
 }
 
 // Tokenize a `key=value [key="quoted value"] ...` option string. Bare
@@ -130,6 +147,41 @@ function parseAssetSyntax(s: string): PipelineTriggerAsset | undefined {
 	return undefined
 }
 
+// Mirror of Rust `parse_asset_syntax(s, enable_default_syntax=true)`: the bare
+// words `ducklake` / `datatable` are shorthand for their `…://main` form. Used
+// by `// materialize` (but NOT by `// on`, which is default-syntax off, so the
+// trigger parser keeps using `parseAssetSyntax`).
+function parseAssetSyntaxDefault(s: string): PipelineTriggerAsset | undefined {
+	if (s === 'datatable') return { kind: 'datatable', path: 'main' }
+	if (s === 'ducklake') return { kind: 'ducklake', path: 'main' }
+	return parseAssetSyntax(s)
+}
+
+// Parse a `// materialize [manual] <asset> [append] [key=<col>]` right-hand
+// side. Optional leading `manual` word opts out of managed mode; the next token
+// is the target asset URI (default-syntax shorthands enabled); the remainder
+// are strategy options (`append` flag, `key=<col>`). Missing/empty target →
+// undefined (dropped).
+function parseMaterializeSpec(s: string): MaterializeSpec | undefined {
+	let manual = false
+	let rest = s
+	const afterManual = consumeKeyword(s, 'manual')
+	if (afterManual !== undefined) {
+		manual = true
+		rest = afterManual.trimStart()
+	}
+	rest = rest.trim()
+	const m = rest.match(/^(\S+)(?:\s+(.*))?$/)
+	if (!m) return undefined
+	const asset = parseAssetSyntaxDefault(m[1])
+	if (!asset || asset.path === '') return undefined
+	const optsStr = m[2] ?? ''
+	const append = optsStr.split(/\s+/).some((t) => t === 'append')
+	const key = parseKvOpts(optsStr).get('key')
+	const uniqueKey = key && key !== '' ? key : undefined
+	return { targetKind: asset.kind, targetPath: asset.path, manual, append, uniqueKey }
+}
+
 type ParsedTriggerSpec =
 	| { kind: 'asset'; value: PipelineTriggerAsset }
 	| { kind: 'native'; value: PipelineNativeTrigger }
@@ -286,6 +338,15 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations {
 			continue
 		}
 
+		const afterMaterialize = consumeKeyword(inner, 'materialize')
+		if (afterMaterialize !== undefined) {
+			if (!out.materialize) {
+				const spec = parseMaterializeSpec(afterMaterialize.trim())
+				if (spec) out.materialize = spec
+			}
+			continue
+		}
+
 		const afterOn = consumeKeyword(inner, 'on')
 		if (afterOn !== undefined) {
 			const specText = afterOn.trim()

frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts

@@ -3,14 +3,22 @@ import { random_adj } from '$lib/components/random_positive_adjetive'
 import { parseDbInputFromAssetSyntax } from '$lib/utils'
 
 // What kind of asset the new script will produce. Drives the random output
-// path scheme and the body skeleton. The output asset is NOT declared in a
-// comment annotation — it's reconstructed from the body's SDK calls / SQL by
-// the asset parser, same as production scripts, so it can't go stale.
+// path scheme and the body skeleton. For most kinds the output asset is NOT
+// declared in a comment annotation — it's reconstructed from the body's SDK
+// calls / SQL by the asset parser, same as production scripts, so it can't go
+// stale. The exception is `materialize`, which declares its target explicitly
+// via the `// materialize` annotation (the runtime generates the write).
 //
 // `none` is the conservative default — body just has a "fill in" comment. The
 // other kinds inject their respective wmill SDK calls / SQL setup so the
 // script is runnable (modulo schema definition) the moment it's created.
-export type PipelineOutputKind = 'none' | 'datatable' | 'ducklake' | 's3_parquet' | 's3_object'
+export type PipelineOutputKind =
+	| 'none'
+	| 'datatable'
+	| 'ducklake'
+	| 'materialize'
+	| 's3_parquet'
+	| 's3_object'
 
 export type PipelineOutputKindMeta = {
 	id: PipelineOutputKind
@@ -23,6 +31,11 @@ export type PipelineOutputKindMeta = {
 // object are the escape hatches for arbitrary blobs; none is last because
 // picking it disables the whole "auto-generated output" feature.
 export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [
+	{
+		id: 'materialize',
+		label: 'Materialized table',
+		description: 'Managed DuckLake table — idempotent, versioned, tracked'
+	},
 	{
 		id: 'datatable',
 		label: 'Data table',
@@ -31,7 +44,7 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [
 	{
 		id: 'ducklake',
 		label: 'Ducklake',
-		description: 'DuckDB lakehouse table'
+		description: 'DuckDB lakehouse table (raw write)'
 	},
 	{
 		id: 's3_parquet',
@@ -60,7 +73,11 @@ const LANG_COMPATIBILITY: Record<ScriptLang, PipelineOutputKind[]> = {
 	bun: ['datatable', 'ducklake', 's3_parquet', 's3_object', 'none'],
 	deno: ['datatable', 'ducklake', 's3_parquet', 's3_object', 'none'],
 	python3: ['datatable', 'ducklake', 's3_parquet', 's3_object', 'none'],
-	duckdb: ['datatable', 'ducklake', 's3_parquet', 's3_object', 'none'],
+	// `materialize` is DuckDB-only: it generates the managed write around a
+	// single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL
+	// SELECT (not in-memory rows), so a polyglot managed materialize is a
+	// separate follow-up — those langs keep the `ducklake` raw-write kind.
+	duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'none'],
 	postgresql: ['datatable', 'none'],
 	mysql: ['none'],
 	mssql: ['none'],
@@ -135,6 +152,7 @@ export function autoOutputAsset(
 		case 'datatable':
 			return { kind: 'datatable', path: `main/${adj}_${pick(TABLE_NOUNS)}_${slug}` }
 		case 'ducklake':
+		case 'materialize':
 			return { kind: 'ducklake', path: `main/${adj}_${pick(TABLE_NOUNS)}_${slug}` }
 		// s3 paths carry the canonical leading slash of a default-storage
 		// object (`s3:///<key>` parses to path `/<key>`). The deploy-time
@@ -286,7 +304,8 @@ export type TemplateContext = {
 // Header: `// pipeline` + every trigger source as its own annotation line.
 // Output asset is NOT declared here — it's reconstructed from the body's
 // SDK calls / SQL by the asset parser, same as production scripts.
-function header(language: ScriptLang, triggers: DraftTriggerSource[]): string {
+function header(ctx: TemplateContext): string {
+	const { language, triggers, output, outputKind } = ctx
 	const p = commentPrefix(language)
 	const lines = triggers.map((t) => {
 		switch (t.kind) {
@@ -298,6 +317,19 @@ function header(language: ScriptLang, triggers: DraftTriggerSource[]): string {
 				return `${p} on ${t.kind}`
 		}
 	})
+	// Managed materialization is the one kind that declares its output
+	// explicitly — the runtime generates the write around the body's SELECT, so
+	// the target can't be inferred from the body. Emit `// materialize <uri>`
+	// plus a hint about the strategy options that go on the same line. The hint
+	// must NOT start with a parser keyword (`materialize`, `on`, …) or it would
+	// be read as an annotation — `Strategy:` is safe.
+	const matLine =
+		outputKind === 'materialize' && output
+			? [
+					`${p} materialize ${assetUri(output)}`,
+					`${p} Strategy: add key=<col> to merge (upsert), or append for insert-only; default replaces the partition`
+				]
+			: []
 	// Discoverability hint — the three annotations users most often miss
 	// when authoring their first pipeline script. Single line, real
 	// example values (not placeholders) so users see the syntax. Docs
@@ -305,7 +337,7 @@ function header(language: ScriptLang, triggers: DraftTriggerSource[]): string {
 	// line separates it from the parsed annotations above (`// pipeline`,
 	// `// on …`) so the editor reads as "real annotations, then a hint".
 	const more = `${p} More: partitioned daily, freshness 1h, retry 3, tag heavy — https://www.windmill.dev/docs/pipelines/annotations`
-	return [`${p} pipeline`, ...lines, '', more, ''].join('\n')
+	return [`${p} pipeline`, ...lines, ...matLine, '', more, ''].join('\n')
 }
 
 // Bun / Deno bodies. These share the wmill SDK surface, so we treat them
@@ -341,27 +373,24 @@ function bodyTs(ctx: TemplateContext): string {
 		switch (input.kind) {
 			case 's3object':
 				return [
-					`  // Upstream: ${assetUri(input)}`,
 					`  const buf = await wmill.loadS3File({ s3: ${JSON.stringify(s3Key(input.path))} })`,
 					`  const rows = JSON.parse(new TextDecoder().decode(buf))`,
 					``
 				].join('\n')
 			case 'datatable':
 				return [
-					`  // Upstream: ${assetUri(input)}`,
 					`  const src = wmill.datatable(${JSON.stringify(input.path.split('/')[0] ?? 'main')})`,
 					`  const rows = await src\`SELECT * FROM ${input.path.split('/').slice(1).join('_') || 'table_name'}\`.fetch()`,
 					``
 				].join('\n')
 			case 'ducklake':
 				return [
-					`  // Upstream: ${assetUri(input)}`,
 					`  const lake = wmill.ducklake(${JSON.stringify(input.path.split('/')[0] ?? 'main')})`,
 					`  const rows = await lake\`SELECT * FROM ${input.path.split('/').slice(1).join('_') || 'table_name'}\`.fetch()`,
 					``
 				].join('\n')
 			default:
-				return `  // Upstream: ${assetUri(input)}\n`
+				return ''
 		}
 	})()
 
@@ -427,24 +456,21 @@ function bodyPython(ctx: TemplateContext): string {
 		switch (input.kind) {
 			case 's3object':
 				return [
-					`    # Upstream: ${assetUri(input)}`,
 					`    buf = wmill.load_s3_file(${JSON.stringify(s3Key(input.path))})`,
 					`    import json; rows = json.loads(buf.decode("utf-8"))`
 				].join('\n')
 			case 'datatable':
 				return [
-					`    # Upstream: ${assetUri(input)}`,
 					`    src = wmill.datatable(${JSON.stringify(input.path.split('/')[0] ?? 'main')})`,
 					`    rows = src.query("SELECT * FROM ${input.path.split('/').slice(1).join('_') || 'table_name'}").fetch()`
 				].join('\n')
 			case 'ducklake':
 				return [
-					`    # Upstream: ${assetUri(input)}`,
 					`    lake = wmill.ducklake(${JSON.stringify(input.path.split('/')[0] ?? 'main')})`,
 					`    rows = lake.query("SELECT * FROM ${input.path.split('/').slice(1).join('_') || 'table_name'}").fetch()`
 				].join('\n')
 			default:
-				return `    # Upstream: ${assetUri(input)}`
+				return ''
 		}
 	})()
 
@@ -498,7 +524,6 @@ function bodyDuckdb(ctx: TemplateContext): string {
 		lines.push(`-- $file (s3object)`)
 		lines.push(`-- \`file\` is uploaded via the S3 picker on the run form.`)
 	}
-	if (input) lines.push(`-- Upstream: ${assetUri(input)}`)
 	lines.push('')
 
 	// Resolve the catalog db name to ATTACH. Output's db wins when both sides
@@ -536,9 +561,9 @@ function bodyDuckdb(ctx: TemplateContext): string {
 			case 'datatable':
 				// `pg` is the attached Postgres catalog (see ATTACH above).
 				// Use a 2-part `pg.<table>` ref so the asset parser maps it
-				// back to `datatable://<db>/<table>` — matching the
-				// `// Upstream` annotation. Schema is only emitted if the
-				// asset path explicitly includes one (`main/myschema.mytable`).
+				// back to `datatable://<db>/<table>` — matching the input asset.
+				// Schema is only emitted if the asset path explicitly includes
+				// one (`main/myschema.mytable`).
 				return `pg.${catalogTableRef(input.path)}`
 			case 'ducklake':
 				return `lake.${catalogTableRef(input.path)}`
@@ -579,6 +604,15 @@ function bodyDuckdb(ctx: TemplateContext): string {
 				)
 			}
 			break
+		case 'materialize':
+			// Managed materialization: the body is just the SELECT that produces
+			// the slice — the runtime wraps it into the idempotent write +
+			// snapshot (see the `// materialize` annotation in the header). No
+			// CREATE TABLE / INSERT, and the target is NOT attached here (the
+			// runtime attaches it). Add `// partitioned daily` + a `{partition}`
+			// filter for a partitioned table.
+			lines.push(`SELECT * FROM ${inSql ?? '(SELECT 1 AS placeholder)'};`)
+			break
 		case 'datatable':
 			if (output) {
 				// 2-part `pg.<table>` so the asset parser resolves the
@@ -605,9 +639,8 @@ function bodyDuckdb(ctx: TemplateContext): string {
 }
 
 function bodyPostgres(ctx: TemplateContext): string {
-	const { input, output, outputKind } = ctx
+	const { output, outputKind } = ctx
 	const lines: string[] = []
-	if (input) lines.push(`-- Upstream: ${assetUri(input)}`)
 	lines.push('')
 
 	if (outputKind === 'datatable' && output) {
@@ -636,9 +669,8 @@ function bodyPostgres(ctx: TemplateContext): string {
 }
 
 function bodyBash(ctx: TemplateContext): string {
-	const { input, output, outputKind } = ctx
+	const { output, outputKind } = ctx
 	const lines: string[] = []
-	if (input) lines.push(`# Upstream: ${assetUri(input)}`)
 	if (outputKind === 's3_object' && output) {
 		lines.push(
 			``,
@@ -655,7 +687,6 @@ function bodyBash(ctx: TemplateContext): string {
 function genericBody(ctx: TemplateContext): string {
 	const p = commentPrefix(ctx.language)
 	const lines: string[] = []
-	if (ctx.input) lines.push(`${p} Upstream: ${assetUri(ctx.input)}`)
 	lines.push(`${p} Fill in pipeline logic.`)
 	return lines.join('\n') + '\n'
 }
@@ -664,7 +695,7 @@ function genericBody(ctx: TemplateContext): string {
 // The returned content is ready to drop into a Script as `content` — no
 // further mutation needed, including for the trigger annotations.
 export function generatePipelineDraft(ctx: TemplateContext): string {
-	const head = header(ctx.language, ctx.triggers)
+	const head = header(ctx)
 	const body = (() => {
 		switch (ctx.language) {
 			case 'bun':

frontend/src/lib/components/dbOps.ts

@@ -45,12 +45,16 @@ export function dbTableOpsWithPreviewScripts({
 	input,
 	tableKey,
 	colDefs,
-	workspace
+	workspace,
+	whereClause
 }: {
 	input: DbInput
 	tableKey: string
 	colDefs: ColumnDef[]
 	workspace: string
+	// Optional raw SQL predicate AND-ed into the read queries (count + rows).
+	// Caller-trusted — build it with escaped values.
+	whereClause?: string
 }): IDbTableOps {
 	const dbType = getDbType(input)
 	const language = getLanguageByResourceType(dbType)
@@ -67,7 +71,11 @@ export function dbTableOpsWithPreviewScripts({
 		tableKey,
 		colDefs,
 		getCount: async ({ quicksearch }) => {
-			const content = makeMarker('COUNT', { table: tableKey, columnDefs: colDefs })
+			const content = makeMarker('COUNT', {
+				table: tableKey,
+				columnDefs: colDefs,
+				...(whereClause ? { whereClause } : {})
+			})
 			const result = await runScriptAndPollResult({
 				workspace,
 				requestBody: { args: { ...dbArg, quicksearch }, language, content }
@@ -79,7 +87,8 @@ export function dbTableOpsWithPreviewScripts({
 			const content = makeMarker('SELECT', {
 				table: tableKey,
 				columnDefs: colDefs,
-				fixPgIntTypes: true
+				fixPgIntTypes: true,
+				...(whereClause ? { whereClause } : {})
 			})
 			let items = (await runScriptAndPollResult({
 				workspace,

python-client/wmill/wmill/client.py

@@ -2289,6 +2289,131 @@ class DucklakeClient:
             )
         )
 
+    def _qualified(self, table: str, schema: str = None) -> str:
+        return f'dl."{schema}"."{table}"' if schema else f"dl.{table}"
+
+    def _materialize_finish(self, sql, table, schema, partition, partition_col):
+        """Return the materialize query; in a pipeline (WM_PIPELINE) append a
+        summary read and record materialized_partition state after a successful
+        run so SDK-materialized slices appear in the grid like `// materialize`
+        ones. Outside a pipeline it stays a plain query (no recording)."""
+        bind = {} if partition is None else {"_wm_partition": partition}
+        if os.environ.get("WM_PIPELINE") != "true":
+            return self.query(sql, **bind)
+        t = self._qualified(table, schema)
+        where = f" WHERE {partition_col} = $_wm_partition" if partition is not None else ""
+        summary = (
+            f"\nSELECT (SELECT count(*) FROM {t}{where}) AS rows, "
+            f"(SELECT max(snapshot_id) FROM ducklake_snapshots('dl')) AS snapshot_id;"
+        )
+        q = self.query(sql + summary, **bind)
+        # Asset path mirrors the `// materialize` engine: <lake>/<schema>.<table>
+        # for an explicit schema, else <lake>/<table>. Dropping the schema would
+        # hide the row from the grid and collide distinct schemas under one key.
+        asset_path = f"{self.name}/{schema}.{table}" if schema else f"{self.name}/{table}"
+        return _RecordingSqlQuery(q, self.client, asset_path, partition or "")
+
+    def upsert_partition(
+        self,
+        table: str,
+        select_sql: str,
+        partition: str = None,
+        unique_key: str = None,
+        partition_col: str = "_wm_partition",
+        schema: str = None,
+    ):
+        """Idempotently materialize the rows of `select_sql` into ducklake
+        `table` for one `partition` (or the whole table when `partition` is
+        None). Client-side equivalent of the `// materialize` engine: with
+        `unique_key` it upserts within the slice (delete-by-key + insert);
+        without it, it replaces (whole table → CREATE OR REPLACE; partition →
+        delete the partition + insert). Re-running the same slice is safe — the
+        backfill / failure-recovery contract.
+
+        The partition value is bound as a DuckDB arg (never string-interpolated)
+        so it cannot inject SQL. `select_sql` is trusted (your own query).
+        """
+        t = self._qualified(table, schema)
+        # Whole-table (no partition): no partition column; replace rebuilds the
+        # table with CREATE OR REPLACE, merge upserts the whole table by key.
+        if partition is None:
+            if unique_key:
+                sql = (
+                    f"CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({select_sql}) WHERE false;\n"
+                    f"BEGIN TRANSACTION;\n"
+                    f"DELETE FROM {t} WHERE {unique_key} IN (SELECT {unique_key} FROM ({select_sql}));\n"
+                    f"INSERT INTO {t} SELECT * FROM ({select_sql});\n"
+                    f"COMMIT;"
+                )
+            else:
+                sql = f"CREATE OR REPLACE TABLE {t} AS SELECT * FROM ({select_sql});"
+            return self._materialize_finish(sql, table, schema, partition, partition_col)
+        src = f"SELECT *, $_wm_partition AS {partition_col} FROM ({select_sql})"
+        if unique_key:
+            # Upsert via delete-by-key + insert (not MERGE — DuckLake's MERGE
+            # fails writing the first rows of a fresh partition).
+            body = (
+                f"DELETE FROM {t} WHERE {partition_col} = $_wm_partition "
+                f"AND {unique_key} IN (SELECT {unique_key} FROM ({select_sql}));\n"
+                f"INSERT INTO {t} {src};"
+            )
+        else:
+            body = (
+                f"DELETE FROM {t} WHERE {partition_col} = $_wm_partition;\n"
+                f"INSERT INTO {t} {src};"
+            )
+        sql = (
+            f"CREATE TABLE IF NOT EXISTS {t} AS "
+            f"SELECT *, CAST(NULL AS VARCHAR) AS {partition_col} FROM ({select_sql}) WHERE false;\n"
+            f"ALTER TABLE {t} SET PARTITIONED BY ({partition_col});\n"
+            f"BEGIN TRANSACTION;\n{body}\nCOMMIT;"
+        )
+        return self._materialize_finish(sql, table, schema, partition, partition_col)
+
+    def append_partition(
+        self,
+        table: str,
+        select_sql: str,
+        partition: str = None,
+        partition_col: str = "_wm_partition",
+        schema: str = None,
+    ):
+        """INSERT-only materialization (no dedup / no replace) for an immutable
+        event-log table — for one `partition`, or the whole table when
+        `partition` is None. NOTE: unlike `upsert_partition`, re-running the same
+        slice duplicates rows — use only for append-only sources."""
+        t = self._qualified(table, schema)
+        # Whole-table (no partition): insert into the bare table, no partition col.
+        if partition is None:
+            sql = (
+                f"CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({select_sql}) WHERE false;\n"
+                f"INSERT INTO {t} SELECT * FROM ({select_sql});"
+            )
+            return self._materialize_finish(sql, table, schema, partition, partition_col)
+        sql = (
+            f"CREATE TABLE IF NOT EXISTS {t} AS "
+            f"SELECT *, CAST(NULL AS VARCHAR) AS {partition_col} FROM ({select_sql}) WHERE false;\n"
+            f"ALTER TABLE {t} SET PARTITIONED BY ({partition_col});\n"
+            f"INSERT INTO {t} SELECT *, $_wm_partition AS {partition_col} FROM ({select_sql});"
+        )
+        return self._materialize_finish(sql, table, schema, partition, partition_col)
+
+    def read(
+        self,
+        table: str,
+        partition: str = None,
+        partition_col: str = "_wm_partition",
+        schema: str = None,
+    ):
+        """Read a materialized ducklake table, optionally a single partition."""
+        t = self._qualified(table, schema)
+        if partition is not None:
+            return self.query(
+                f"SELECT * FROM {t} WHERE {partition_col} = $_wm_partition",
+                _wm_partition=partition,
+            )
+        return self.query(f"SELECT * FROM {t}")
+
 class SqlQuery:
     """Query result handler for DataTable and DuckLake queries."""
 
@@ -2337,6 +2462,60 @@ class SqlQuery:
         """
         self.fetch_one()
 
+
+class _RecordingSqlQuery:
+    """Wraps a ducklake materialize query so that, on a successful run, the
+    trailing summary (row count + snapshot id) is captured and the
+    materialized_partition state is recorded (best-effort). Only used in pipeline
+    context — outside it the helpers return a plain SqlQuery. Mirrors SqlQuery's
+    terminal methods so `.execute()` / `.fetch_one()` behave the same."""
+
+    def __init__(self, inner, client, asset_path, partition):
+        self._inner = inner
+        self._client = client
+        self._asset_path = asset_path
+        self._partition = partition
+        self.sql = inner.sql
+
+    def execute(self):
+        self._run()
+
+    def fetch_one(self):
+        return self._run()
+
+    def fetch(self, result_collection=None):
+        return self._run()
+
+    def _run(self):
+        try:
+            row = self._inner.fetch_one()
+        except Exception as e:
+            self._record("failed", None, None, str(e))
+            raise
+        snap = row.get("snapshot_id") if isinstance(row, dict) else None
+        rows = row.get("rows") if isinstance(row, dict) else None
+        self._record("materialized", snap, rows, None)
+        return row
+
+    def _record(self, status, snapshot_id, row_count, error):
+        try:
+            self._client.post(
+                f"/w/{self._client.workspace}/assets/record_materialization",
+                json={
+                    "asset_kind": "ducklake",
+                    "asset_path": self._asset_path,
+                    "partition": self._partition,
+                    "status": status,
+                    "snapshot_id": snapshot_id,
+                    "row_count": row_count,
+                    "job_id": os.environ.get("WM_JOB_ID"),
+                    "error": error,
+                },
+            )
+        except Exception:
+            pass  # best-effort; never fail the user's materialization
+
+
 def infer_sql_type(value) -> str:
     """
     DuckDB executor requires explicit argument types at declaration

system_prompts/auto-generated/prompts.ts

@@ -1487,6 +1487,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize \`selectSql\` into a ducklake table for one
+ * partition (or the whole table when \`partition\` is omitted) — the client-side
+ * equivalent of the \`// materialize\` engine.
+ * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call \`.execute()\` to run it:
+ * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
 `;
 
 export const SDK_PYTHON = `# Python SDK (wmill)
@@ -2043,6 +2066,27 @@ def stream_result(stream) -> None
 #     SqlQuery instance for fetching results
 def query(sql: str, *args) -> SqlQuery
 
+# Idempotently materialize the rows of \`select_sql\` into ducklake
+# \`table\` for one \`partition\` (or the whole table when \`partition\` is
+# None). Client-side equivalent of the \`// materialize\` engine: with
+# \`unique_key\` it upserts within the slice (delete-by-key + insert);
+# without it, it replaces (whole table → CREATE OR REPLACE; partition →
+# delete the partition + insert). Re-running the same slice is safe — the
+# backfill / failure-recovery contract.
+# 
+# The partition value is bound as a DuckDB arg (never string-interpolated)
+# so it cannot inject SQL. \`select_sql\` is trusted (your own query).
+def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# INSERT-only materialization (no dedup / no replace) for an immutable
+# event-log table — for one \`partition\`, or the whole table when
+# \`partition\` is None. NOTE: unlike \`upsert_partition\`, re-running the same
+# slice duplicates rows — use only for append-only sources.
+def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# Read a materialized ducklake table, optionally a single partition.
+def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
 # Execute query and fetch results.
 # 
 # Args:

system_prompts/auto-generated/script.md

@@ -1991,6 +1991,29 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  */
 ducklake(name: string = "main"): SqlTemplateFunction
 
+/**
+ * Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>
+
 
 # Python SDK (wmill)
 
@@ -2546,6 +2569,27 @@ def stream_result(stream) -> None
 #     SqlQuery instance for fetching results
 def query(sql: str, *args) -> SqlQuery
 
+# Idempotently materialize the rows of `select_sql` into ducklake
+# `table` for one `partition` (or the whole table when `partition` is
+# None). Client-side equivalent of the `// materialize` engine: with
+# `unique_key` it upserts within the slice (delete-by-key + insert);
+# without it, it replaces (whole table → CREATE OR REPLACE; partition →
+# delete the partition + insert). Re-running the same slice is safe — the
+# backfill / failure-recovery contract.
+# 
+# The partition value is bound as a DuckDB arg (never string-interpolated)
+# so it cannot inject SQL. `select_sql` is trusted (your own query).
+def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# INSERT-only materialization (no dedup / no replace) for an immutable
+# event-log table — for one `partition`, or the whole table when
+# `partition` is None. NOTE: unlike `upsert_partition`, re-running the same
+# slice duplicates rows — use only for append-only sources.
+def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# Read a materialized ducklake table, optionally a single partition.
+def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
 # Execute query and fetch results.
 # 
 # Args:

system_prompts/auto-generated/sdks/python.md

@@ -552,6 +552,27 @@ def stream_result(stream) -> None
 #     SqlQuery instance for fetching results
 def query(sql: str, *args) -> SqlQuery
 
+# Idempotently materialize the rows of `select_sql` into ducklake
+# `table` for one `partition` (or the whole table when `partition` is
+# None). Client-side equivalent of the `// materialize` engine: with
+# `unique_key` it upserts within the slice (delete-by-key + insert);
+# without it, it replaces (whole table → CREATE OR REPLACE; partition →
+# delete the partition + insert). Re-running the same slice is safe — the
+# backfill / failure-recovery contract.
+# 
+# The partition value is bound as a DuckDB arg (never string-interpolated)
+# so it cannot inject SQL. `select_sql` is trusted (your own query).
+def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# INSERT-only materialization (no dedup / no replace) for an immutable
+# event-log table — for one `partition`, or the whole table when
+# `partition` is None. NOTE: unlike `upsert_partition`, re-running the same
+# slice duplicates rows — use only for append-only sources.
+def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# Read a materialized ducklake table, optionally a single partition.
+def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
 # Execute query and fetch results.
 # 
 # Args:

system_prompts/auto-generated/sdks/typescript.md

@@ -557,3 +557,26 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>

system_prompts/auto-generated/skills/write-script-bun/SKILL.md

@@ -728,3 +728,26 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>

system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md

@@ -728,3 +728,26 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>

system_prompts/auto-generated/skills/write-script-deno/SKILL.md

@@ -728,3 +728,26 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 ducklake(name: string = "main"): SqlTemplateFunction
+
+/**
+ * Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`.
+ */
+upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>
+
+/**
+ * INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ * 
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`.
+ */
+appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,): SqlStatement<any>

system_prompts/auto-generated/skills/write-script-python3/SKILL.md

@@ -737,6 +737,27 @@ def stream_result(stream) -> None
 #     SqlQuery instance for fetching results
 def query(sql: str, *args) -> SqlQuery
 
+# Idempotently materialize the rows of `select_sql` into ducklake
+# `table` for one `partition` (or the whole table when `partition` is
+# None). Client-side equivalent of the `// materialize` engine: with
+# `unique_key` it upserts within the slice (delete-by-key + insert);
+# without it, it replaces (whole table → CREATE OR REPLACE; partition →
+# delete the partition + insert). Re-running the same slice is safe — the
+# backfill / failure-recovery contract.
+# 
+# The partition value is bound as a DuckDB arg (never string-interpolated)
+# so it cannot inject SQL. `select_sql` is trusted (your own query).
+def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# INSERT-only materialization (no dedup / no replace) for an immutable
+# event-log table — for one `partition`, or the whole table when
+# `partition` is None. NOTE: unlike `upsert_partition`, re-running the same
+# slice duplicates rows — use only for append-only sources.
+def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
+# Read a materialized ducklake table, optionally a single partition.
+def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None)
+
 # Execute query and fetch results.
 # 
 # Args:

typescript-client/build.jsr.sh

@@ -15,6 +15,6 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
 echo "" >> "${script_dirpath}/src/index.ts"
 echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts"
+echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts"
 
 

typescript-client/build.sh

@@ -40,7 +40,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/"
 echo "" >> "${script_dirpath}/src/index.ts"
 echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
 echo "" >> "${script_dirpath}/src/index.ts"
-echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, type TaskOptions, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts"
+echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, type TaskOptions, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts"
 
 # Build default export by combining client utilities + services
 # This preserves backward compatibility for `import wmill from "windmill-client"`
@@ -99,6 +99,8 @@ import {
   streamResult,
   datatable,
   ducklake,
+  upsertPartition,
+  appendPartition,
   SHARED_FOLDER,
   getWorkspace,
   getStatePath,
@@ -185,6 +187,8 @@ const wmill = {
   streamResult,
   datatable,
   ducklake,
+  upsertPartition,
+  appendPartition,
   SHARED_FOLDER,
   getWorkspace,
   getStatePath,

typescript-client/client.d.ts

@@ -22,6 +22,10 @@ export {
 export {
   datatable,
   ducklake,
+  upsertPartition,
+  appendPartition,
+  type DucklakeMaterializeOptions,
+  type SqlStatement,
   type SqlTemplateFunction,
   type DatatableSqlTemplateFunction,
 } from "./sqlUtils";

typescript-client/client.ts

@@ -27,8 +27,12 @@ export {
 export {
   datatable,
   ducklake,
+  upsertPartition,
+  appendPartition,
   type SqlTemplateFunction,
   type DatatableSqlTemplateFunction,
+  type DucklakeMaterializeOptions,
+  type SqlStatement,
 } from "./sqlUtils";
 
 // Services are NOT re-exported here to enable tree-shaking
@@ -71,7 +75,7 @@ function getPublicBaseUrl(): string {
   return getEnv("WM_BASE_URL") ?? "http://localhost:3000";
 }
 
-const getEnv = (key: string) => {
+export const getEnv = (key: string) => {
   if (typeof window === "undefined") {
     // node
     return process?.env?.[key];

typescript-client/sqlUtils.d.ts

@@ -87,3 +87,18 @@ export interface DatatableSqlTemplateFunction extends SqlTemplateFunction {
 
 export declare function datatable(name: string): DatatableSqlTemplateFunction;
 export declare function ducklake(name: string): SqlTemplateFunction;
+
+export interface DucklakeMaterializeOptions {
+  ducklake?: string;
+  table: string;
+  selectSql: string;
+  partition?: string;
+  uniqueKey?: string;
+  partitionCol?: string;
+}
+export declare function upsertPartition(
+  opts: DucklakeMaterializeOptions,
+): SqlStatement<any>;
+export declare function appendPartition(
+  opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,
+): SqlStatement<any>;

typescript-client/sqlUtils.ts

@@ -1,4 +1,5 @@
-import { getWorkspace, workerHasInternalServer } from "./client";
+import { getEnv, getWorkspace, workerHasInternalServer } from "./client";
+import { OpenAPI } from "./core/OpenAPI";
 import { JobService } from "./services.gen";
 
 type ResultCollection =
@@ -385,6 +386,185 @@ export function ducklake(name: string = "main"): SqlTemplateFunction {
   return buildSqlTemplateFunction(ducklakeProvider(n, schema));
 }
 
+/** Options for the ducklake materialization helpers. `partition` is bound as a
+ * DuckDB arg (never interpolated); `selectSql`, `table`, `schema`, `uniqueKey`
+ * are trusted structural SQL inlined via `raw`. */
+export interface DucklakeMaterializeOptions {
+  /** ducklake name (default "main"), optionally "name:schema". */
+  ducklake?: string;
+  /** target table within the ducklake. */
+  table: string;
+  /** the SELECT producing the rows for this slice. */
+  selectSql: string;
+  /** the partition value (bound). Omit for a whole-table materialization — no
+   * partition column, and replace becomes a `CREATE OR REPLACE TABLE`. */
+  partition?: string;
+  /** dedup key → upsert in slice (delete-by-key + insert); omit → replace (delete partition + insert). */
+  uniqueKey?: string;
+  /** physical partition column (default "_wm_partition"). */
+  partitionCol?: string;
+}
+
+/** Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ *
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`. */
+export function upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any> {
+  return finishMaterialize(buildUpsertStatement(opts), opts);
+}
+function buildUpsertStatement(opts: DucklakeMaterializeOptions): SqlStatement<any> {
+  let { name: n, schema } = parseName(opts.ducklake ?? "main");
+  let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+  let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+  let t = sql.raw(`dl.${opts.table}`);
+  let body = sql.raw(opts.selectSql);
+  // Whole-table (no partition): no partition column. Replace rebuilds the table
+  // with CREATE OR REPLACE (handles schema changes); merge upserts by key.
+  if (opts.partition === undefined) {
+    if (opts.uniqueKey) {
+      let uk = sql.raw(opts.uniqueKey);
+      return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT * FROM (${body});
+COMMIT;`;
+    }
+    return sql`CREATE OR REPLACE TABLE ${t} AS SELECT * FROM (${body});`;
+  }
+  if (opts.uniqueKey) {
+    let uk = sql.raw(opts.uniqueKey);
+    // Upsert via delete-by-key + insert (not MERGE — DuckLake's MERGE fails
+    // writing the first rows of a fresh partition).
+    return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition} AND ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+  }
+  return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition};
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+}
+
+/** INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ *
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`. */
+export function appendPartition(
+  opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,
+): SqlStatement<any> {
+  return finishMaterialize(buildAppendStatement(opts), opts);
+}
+function buildAppendStatement(
+  opts: Omit<DucklakeMaterializeOptions, "uniqueKey">,
+): SqlStatement<any> {
+  let { name: n, schema } = parseName(opts.ducklake ?? "main");
+  let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+  let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+  let t = sql.raw(`dl.${opts.table}`);
+  let body = sql.raw(opts.selectSql);
+  // Whole-table (no partition): insert into the bare table, no partition column.
+  if (opts.partition === undefined) {
+    return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+INSERT INTO ${t} SELECT * FROM (${body});`;
+  }
+  return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});`;
+}
+
+// In pipeline context (WM_PIPELINE), wrap a materialize statement so a
+// successful run captures the slice's row count + snapshot and records
+// materialized_partition state — making SDK materializations appear in the grid
+// like `// materialize` ones. Outside a pipeline it's a passthrough (no record).
+function finishMaterialize(
+  stmt: SqlStatement<any>,
+  opts: Pick<DucklakeMaterializeOptions, "ducklake" | "table" | "partition" | "partitionCol">,
+): SqlStatement<any> {
+  if (getEnv("WM_PIPELINE") !== "true") return stmt;
+  let { name: n, schema } = parseName(opts.ducklake ?? "main");
+  let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+  let t = sql.raw(`dl.${opts.table}`);
+  let pcol = opts.partitionCol ?? "_wm_partition";
+  let where =
+    opts.partition !== undefined
+      ? sql.raw(`WHERE ${pcol} = '${String(opts.partition).replace(/'/g, "''")}'`)
+      : sql.raw("");
+  let summary = sql`SELECT (SELECT count(*) FROM ${t} ${where}) AS rows, (SELECT max(snapshot_id) FROM ducklake_snapshots('dl')) AS snapshot_id`;
+  // Asset path mirrors the `// materialize` engine: <lake>/<schema>.<table> for
+  // an explicit schema, else <lake>/<table> — so the grid lookup matches and
+  // distinct schemas don't collide under one state key.
+  let assetPath = schema ? `${n}/${schema}.${opts.table}` : `${n}/${opts.table}`;
+  let partition = opts.partition ?? "";
+  let run = async () => {
+    try {
+      await stmt.execute();
+    } catch (e) {
+      await recordMaterialization(assetPath, partition, "failed", null, null, String(e));
+      throw e;
+    }
+    let snapshot_id: number | null = null;
+    let row_count: number | null = null;
+    try {
+      let s: any = await summary.fetchOne();
+      snapshot_id = s?.snapshot_id ?? null;
+      row_count = s?.rows ?? null;
+    } catch {
+      /* summary read is best-effort */
+    }
+    await recordMaterialization(assetPath, partition, "materialized", snapshot_id, row_count, null);
+  };
+  return {
+    ...stmt,
+    execute: (() => run()) as any,
+    fetch: (() => run()) as any,
+    fetchOne: (() => run()) as any,
+    fetchOneScalar: (() => run()) as any,
+  };
+}
+
+async function recordMaterialization(
+  assetPath: string,
+  partition: string,
+  status: string,
+  snapshot_id: number | null,
+  row_count: number | null,
+  error: string | null,
+): Promise<void> {
+  try {
+    await fetch(`${OpenAPI.BASE}/w/${getWorkspace()}/assets/record_materialization`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: `Bearer ${OpenAPI.TOKEN as string}`,
+      },
+      body: JSON.stringify({
+        asset_kind: "ducklake",
+        asset_path: assetPath,
+        partition,
+        status,
+        snapshot_id,
+        row_count,
+        job_id: getEnv("WM_JOB_ID") ?? null,
+        error,
+      }),
+    });
+  } catch {
+    // best-effort; never fail the user's materialization
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Utilities
 // ---------------------------------------------------------------------------