feat: define a materialized view by its query, with Functions in FROM position (#4190)

A view definition was a structured record under a `kind` tag, one kind
per query shape, and a Function returning `list<struct>` was about to
add a third. That names shapes instead of describing a relation.

A materialized view is now a relation defined by a query, stored as one
canonical SQL string under a format number:

```sql
SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [LIMIT n]
```

Any other clause is refused at parse time. Older readers report the view
as unrefreshable, the pre-format layouts still read, and a legacy view
is rewritten on its next refresh that commits, rebuilt only where its
raw text meant something else under lance's parser.

A Function in FROM position yields one row per element it returns, as a
table function does in any dialect. The server stages its list output in
a hidden table and records that binding beside the query, which stays as
the user wrote it; refresh scans the staging and unnests the column, the
same operator as UNNEST over a list column the table already holds. Row
ids repeat per element, so eviction and incremental append are
unchanged. A local database refuses a Function in FROM position, since
it has no executor.

nodejs/__test__/materialized_view.test.ts

@@ -28,6 +28,39 @@ describe("materialized views", () => {
   });
   afterEach(() => tmpDir.removeCallback());
 
+  it("reads stored queries and legacy layouts", () => {
+    const read = (stored: string) =>
+      definitionFromMetadata(new Map([[DEFINITION_META_KEY, stored]]), "v");
+    const query =
+      "SELECT id, c.chunk FROM ns.docs, UNNEST(chunks) AS c WHERE id > 1";
+    expect(read(`{"format":1,"query":${JSON.stringify(query)}}`).query).toBe(
+      query,
+    );
+
+    // The structured layout written before the format number reads as the
+    // query it described, under either of its kind tags.
+    expect(
+      read(
+        '{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"],' +
+          '"projections":[{"output":"name","expression":"`name`"},' +
+          '{"output":"Shout","expression":"upper(name)"}],"filter":"age >= 18","limit":42}',
+      ).query,
+    ).toBe(
+      "SELECT `name`, upper(name) AS `Shout` FROM ns.people WHERE age >= 18 LIMIT 42",
+    );
+    expect(read('{"kind":"select","source_table":"people"}').query).toBe(
+      "SELECT * FROM people",
+    );
+
+    // A newer writer's layout is reported, never guessed at.
+    for (const newer of [
+      `{"format":2,"query":${JSON.stringify(query)}}`,
+      '{"kind":"select_v3","source_table":"people"}',
+    ]) {
+      expect(() => read(newer)).toThrow(/cannot refresh/);
+    }
+  });
+
   it("rejects a stored limit a number cannot carry", () => {
     const big = new Map([
       [
@@ -38,36 +71,6 @@ describe("materialized views", () => {
     expect(() => definitionFromMetadata(big, "v")).toThrow(
       /too large to represent exactly/,
     );
-
-    const safe = new Map([
-      [
-        DEFINITION_META_KEY,
-        '{"kind":"select","source_table":"people","limit":42}',
-      ],
-    ]);
-    expect(definitionFromMetadata(safe, "v").limit).toBe(42);
-  });
-
-  it("reads the namespaced select kind and refuses unknown kinds", () => {
-    // "namespaced_select" is the namespaced form of "select": same shape, a
-    // separate kind so readers that predate it refuse instead of resolving
-    // the source at the root.
-    const namespaced = new Map([
-      [
-        DEFINITION_META_KEY,
-        '{"kind":"namespaced_select","source_table":"people","source_namespace":["ns"]}',
-      ],
-    ]);
-    const definition = definitionFromMetadata(namespaced, "v");
-    expect(definition.sourceTable).toBe("people");
-    expect(definition.sourceNamespace).toEqual(["ns"]);
-
-    const unknown = new Map([
-      [DEFINITION_META_KEY, '{"kind":"select_v3","source_table":"people"}'],
-    ]);
-    expect(() => definitionFromMetadata(unknown, "v")).toThrow(
-      /cannot refresh/,
-    );
   });
 
   it("creates, refreshes and queries a view", async () => {
@@ -88,13 +91,9 @@ describe("materialized views", () => {
     });
     const view = await db.openMaterializedView("adults");
     const definition = await view.definition();
-    expect(definition.sourceTable).toBe("people");
-    expect(definition.filter).toBe("age >= 18");
-    expect(definition.projections).toEqual([
-      ["name", "`name`"],
-      ["age", "`age`"],
-    ]);
-    expect(definition.inputs).toEqual(["age", "name"]);
+    expect(definition.query).toBe(
+      "SELECT name, age FROM people WHERE age >= 18",
+    );
   });
 
   it("refreshes incrementally after an append", async () => {

nodejs/lancedb/materialized_view.ts

@@ -7,20 +7,18 @@ import { Table } from "./table";
 /** Schema metadata key holding a materialized view's definition. */
 export const DEFINITION_META_KEY = "mv.definition";
 
-/** The query that defines a materialized view. */
+/** The stored layout this version reads: `{"format": 1, "query": "<SQL>"}`. */
+export const DEFINITION_FORMAT = 1;
+
+/**
+ * The query that defines a materialized view, as stored:
+ * `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
+ * [WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row
+ * per element it returns.
+ */
 export interface MaterializedViewDefinition {
-  /** Name of the source table, in the same database as the view. */
-  sourceTable: string;
-  /** `[output column, SQL expression]` pairs, in view schema order. */
-  projections: [string, string][];
-  /** SQL predicate selecting the source rows the view holds. */
-  filter?: string;
-  /** Cap on the number of rows the view holds. */
-  limit?: number;
-  /** Source columns the projections and filter read. */
-  inputs: string[];
-  /** Namespace holding the source table; empty is the root namespace. */
-  sourceNamespace: string[];
+  /** The defining query, in the canonical spelling the server stores. */
+  query: string;
 }
 
 /**
@@ -88,15 +86,21 @@ export function definitionFromJson(
 ): MaterializedViewDefinition {
   // biome-ignore lint/suspicious/noExplicitAny: raw JSON
   const value: any = JSON.parse(raw);
-  // "namespaced_select" keeps older readers from resolving the source at root.
-  if (
-    value.kind !== undefined &&
-    value.kind !== "select" &&
-    value.kind !== "namespaced_select"
-  ) {
+  if (value.format !== undefined) {
+    // A newer writer's layout is reported, never guessed at.
+    if (!Number.isInteger(value.format) || value.format > DEFINITION_FORMAT) {
+      throw new Error(
+        `materialized view '${name}' is stored in format ${value.format}, ` +
+          "which this version of lancedb cannot refresh",
+      );
+    }
+    return { query: value.query };
+  }
+  // The structured layout written before the format number.
+  if (value.kind !== "select" && value.kind !== "namespaced_select") {
     throw new Error(
-      `materialized view '${name}' is defined by '${value.kind}', which this ` +
-        "version of lancedb cannot refresh",
+      `materialized view '${name}' is stored in format kind '${value.kind}', ` +
+        "which this version of lancedb cannot refresh",
     );
   }
   const limit = value.limit ?? undefined;
@@ -108,18 +112,41 @@ export function definitionFromJson(
       `materialized view '${name}' has a stored limit too large to represent exactly`,
     );
   }
-  return {
-    sourceTable: value.source_table,
-    // biome-ignore lint/suspicious/noExplicitAny: raw JSON
-    projections: (value.projections ?? []).map((p: any) => [
-      p.output,
-      p.expression,
-    ]),
-    filter: value.filter ?? undefined,
-    limit,
-    inputs: value.inputs ?? [],
-    sourceNamespace: value.source_namespace ?? [],
-  };
+  return { query: legacyQuery(value, limit) };
+}
+
+function legacyIdent(name: string): string {
+  return /^[a-z_][a-z0-9_]*$/.test(name)
+    ? name
+    : `\`${name.replace(/`/g, "``")}\``;
+}
+
+/** Render the pre-format structured layout as the query it described. */
+// biome-ignore lint/suspicious/noExplicitAny: raw JSON
+function legacyQuery(value: any, limit: number | undefined): string {
+  // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+  const projections: any[] = value.projections ?? [];
+  const columns =
+    projections.length === 0
+      ? "*"
+      : projections
+          .map((p) =>
+            p.expression === p.output || p.expression === `\`${p.output}\``
+              ? p.expression
+              : `${p.expression} AS ${legacyIdent(p.output)}`,
+          )
+          .join(", ");
+  const table = [...(value.source_namespace ?? []), value.source_table]
+    .map(legacyIdent)
+    .join(".");
+  let query = `SELECT ${columns} FROM ${table}`;
+  if (value.filter !== undefined && value.filter !== null) {
+    query += ` WHERE ${value.filter}`;
+  }
+  if (limit !== undefined) {
+    query += ` LIMIT ${limit}`;
+  }
+  return query;
 }
 
 /**

nodejs/src/table.rs

@@ -479,7 +479,7 @@ impl Table {
         let view = lancedb::MaterializedView::from_table(inner)
             .await
             .default_error()?;
-        serde_json::to_string(view.definition()).map_err(|err| {
+        view.definition().to_json().map_err(|err| {
             napi::Error::from_reason(format!(
                 "failed to serialize materialized-view definition: {err}"
             ))