Merge remote-tracking branch 'origin/main' into yang/udf-secret-api

BREAKING CHANGE: list_tables now provides tables in arbitrary order and
the page token is now completely opaque. `table_names` retains the old
behavior of lexical ordering and `start-after` semantics.

Listing the tables in a directory database cost what the database held
rather than what the page held. `ListingDatabase::list_tables`
enumerated every child directory of the base path, sorted the names,
then discarded all but the requested page — on every request, for every
page. On object storage that is one full listing per page.

This PR pages the store instead. `list_tables` asks for one page at a
time through `ObjectStore::read_dir_page`, carrying the store's own
continuation token, so a page is one request. Non-table children can
leave a page short of its limit, so the walk continues until the page is
full or the store runs out.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.10"
+current_version = "0.38.0-beta.11"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

Cargo.lock

@@ -5402,7 +5402,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 dependencies = [
  "arrow",
  "async-trait",

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.10</version>
+    <version>0.38.0-beta.11</version>
 </dependency>
 ```
 

docs/src/js/classes/Table.md

@@ -1292,6 +1292,18 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
 
 Update per-field (column) metadata.
 
+The following keys are treated specially, by convention, and should be
+used when appropriate:
+
+- `lancedb:description`: for a human-readable description of a field.
+- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
+  names the tag category; e.g. `lancedb:tag:model: "clip"`.
+- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
+  `feature_v2` might be in the same logical column.
+- `lancedb:status`: for status options (`production`, `candidate`,
+  `deprecated`, `archived`) to designate the current life cycle state of
+  this column.
+
 #### Parameters
 
 * **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]

docs/src/js/interfaces/FieldMetadataUpdate.md

@@ -17,7 +17,8 @@ metadata: Record<string, null | string>;
 ```
 
 Metadata key/value pairs. Merged into the field's existing metadata by
-default; a value of `null` deletes that key.
+default; a value of `null` deletes that key. See
+[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
 
 ***
 

docs/src/python/python.md

@@ -223,9 +223,13 @@ tokens = list(
 Blob columns store large binary values out of line so they can be read lazily
 instead of being materialized with the rest of the row.
 
-::: lancedb.blob
+`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
+pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
+still work. Queries return descriptors. Call
+[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
+[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
 
-::: lancedb.BlobType
+::: lancedb.blob
 
 ::: lancedb._blob.BlobFile
     options:

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.10</version>
+      <version>0.38.0-beta.11</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.10</version>
+    <version>0.38.0-beta.11</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/arrow.test.ts

@@ -1,11 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import * as fs from "node:fs";
+import * as vm from "node:vm";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
+  Field as CurrentField,
+  LargeBinary as CurrentLargeBinary,
+  Schema as CurrentSchema,
   Vector as CurrentVector,
   convertToTable,
   tableFromIPC as currentTableFromIPC,
@@ -36,6 +41,59 @@ function sampleRecords(): Array<Record<string, any>> {
     },
   ];
 }
+
+it("serializes an Arrow Table created in another JavaScript realm", async () => {
+  const context = vm.createContext({
+    TextDecoder,
+    TextEncoder,
+    console,
+    setTimeout,
+    clearTimeout,
+  });
+  vm.runInContext(
+    fs.readFileSync(
+      require.resolve("apache-arrow-15/Arrow.es2015.min"),
+      "utf8",
+    ),
+    context,
+  );
+  const foreignTable: unknown = vm.runInContext(
+    "Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
+    context,
+  );
+
+  const foreignMetadata = (
+    foreignTable as { schema: { metadata: Map<string, string> } }
+  ).schema.metadata;
+  expect(foreignMetadata).not.toBeInstanceOf(Map);
+
+  const buf = await fromDataToBuffer(
+    foreignTable as Parameters<typeof fromDataToBuffer>[0],
+  );
+  const actual = currentTableFromIPC(buf);
+
+  expect(actual.numRows).toBe(3);
+  expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
+  expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
+});
+
+it("preserves field metadata from a provided schema", async function () {
+  const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
+  const schema = new CurrentSchema([
+    new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
+  ]);
+
+  const table = makeArrowTable(
+    [{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
+    { schema },
+  );
+
+  expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
+
+  const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
+  expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
+});
+
 describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Arrow",
   (

nodejs/__test__/embedding.test.ts

@@ -187,6 +187,58 @@ describe("embedding functions", () => {
     const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
     expect(vector0).toEqual([1, 2, 3]);
   });
+  it("should append multiple Python embeddings with the same alias", async () => {
+    @register("python-mock")
+    // biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map((value) =>
+          value === "hello world" ? [1, 2, 3] : [4, 5, 6],
+        );
+      }
+    }
+
+    const metadata = new Map([
+      [
+        "embedding_functions",
+        '[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
+      ],
+    ]);
+    const schema = new Schema(
+      [
+        new Field("text1", new Utf8(), true),
+        new Field("text2", new Utf8(), true),
+        new Field(
+          "vector1",
+          new FixedSizeList(3, new Field("item", new Float32(), true)),
+          true,
+        ),
+        new Field(
+          "vector2",
+          new FixedSizeList(3, new Field("item", new Float32(), true)),
+          true,
+        ),
+      ],
+      metadata,
+    );
+
+    const db = await connect(tmpDir.name);
+    const table = await db.createEmptyTable("test", schema);
+    await table.add([{ text1: "hello world", text2: "goodbye world" }]);
+
+    const rows = await table.query().toArray();
+    expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
+    expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
+  });
 
   it("should append generated vectors to a non-nullable schema", async () => {
     @register("non_nullable_schema_test")

nodejs/__test__/table.test.ts

@@ -3561,6 +3561,27 @@ describe("when creating an empty table", () => {
     expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
   });
 
+  it("can add and query JSON data", async () => {
+    const schema = new Schema([
+      new Field("id", new Int32(), true),
+      new Field(
+        "meta",
+        new Utf8(),
+        true,
+        new Map([["ARROW:extension:name", "arrow.json"]]),
+      ),
+    ]);
+    const table = await con.createEmptyTable("json", schema);
+    const meta = JSON.stringify({ x: 1 });
+
+    await table.add([{ id: 1, meta }]);
+
+    const rows = await table.query().toArray();
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe(1);
+    expect(rows[0].meta).toBe(meta);
+  });
+
   it("can create an empty table from schema that specifies field types by name", async () => {
     const schemaLike = {
       fields: [

nodejs/examples/basic.test.ts

@@ -170,7 +170,7 @@ test("basic table examples", async () => {
     // --8<-- [end:create_index]
 
     // --8<-- [start:delete_rows]
-    await tbl.delete('item = "fizz"');
+    await tbl.delete("item = 'fizz'");
     // --8<-- [end:delete_rows]
 
     // --8<-- [start:drop_table]

nodejs/lancedb/arrow.ts

@@ -72,8 +72,7 @@ export type FieldLike =
     };
 
 export type DataLike =
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  | import("apache-arrow").Data<Struct<any>>
+  | import("apache-arrow").Data
   | {
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       type: any;
@@ -82,6 +81,7 @@ export type DataLike =
       stride: number;
       nullable: boolean;
       children: DataLike[];
+      dictionary?: { data: readonly DataLike[] };
       get nullCount(): number;
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       values: Buffers<any>[BufferType.DATA];

nodejs/lancedb/query.ts

@@ -727,11 +727,11 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
    * Add a query vector to the search
    *
    * This method can be called multiple times to add multiple query vectors
-   * to the search. If multiple query vectors are added, then they will be searched
-   * in parallel, and the results will be concatenated. A column called `query_index`
-   * will be added to indicate the index of the query vector that produced the result.
-   *
-   * Performance wise, this is equivalent to running multiple queries concurrently.
+   * to the search. A column called `query_index` will be added to indicate the index
+   * of the query vector that produced the result. Flat searches share one table scan
+   * across the query vectors, avoiding the scan and memory amplification of running
+   * multiple queries concurrently. Indexed searches may still perform per-vector
+   * index work.
    */
   addQueryVector(vector: IntoVector): VectorQuery {
     if (vector instanceof Promise) {

nodejs/lancedb/sanitize.ts

@@ -94,17 +94,24 @@ export function sanitizeMetadata(
   if (metadataLike === undefined || metadataLike === null) {
     return undefined;
   }
-  if (!(metadataLike instanceof Map)) {
+
+  let entries: IterableIterator<[unknown, unknown]>;
+  try {
+    entries = Map.prototype.entries.call(metadataLike);
+  } catch {
     throw Error("Expected metadata, if present, to be a Map<string, string>");
   }
-  for (const item of metadataLike) {
-    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
+
+  const metadata = new Map<string, string>();
+  for (const [key, value] of entries) {
+    if (typeof key !== "string" || typeof value !== "string") {
       throw Error(
         "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
       );
     }
+    metadata.set(key, value);
   }
-  return metadataLike as Map<string, string>;
+  return metadata;
 }
 
 export function sanitizeInt(typeLike: object) {

nodejs/lancedb/schema.ts

@@ -406,10 +406,11 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
           field.name,
           new Struct(matchingFields(struct.children, value)),
           field.nullable,
+          field.metadata,
         ),
       );
     } else {
-      matches.push(new Field(field.name, value as DataType, field.nullable));
+      matches.push(field);
     }
   }
   return matches;

nodejs/lancedb/table.ts

@@ -630,6 +630,18 @@ export abstract class Table {
 
   /**
    * Update per-field (column) metadata.
+   *
+   * The following keys are treated specially, by convention, and should be
+   * used when appropriate:
+   *
+   * - `lancedb:description`: for a human-readable description of a field.
+   * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
+   *   names the tag category; e.g. `lancedb:tag:model: "clip"`.
+   * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
+   *   `feature_v2` might be in the same logical column.
+   * - `lancedb:status`: for status options (`production`, `candidate`,
+   *   `deprecated`, `archived`) to designate the current life cycle state of
+   *   this column.
    * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
    * update's metadata is merged into the field's existing metadata by default;
    * a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -1555,7 +1567,8 @@ export interface FieldMetadataUpdate {
   path: string;
   /**
    * Metadata key/value pairs. Merged into the field's existing metadata by
-   * default; a value of `null` deletes that key.
+   * default; a value of `null` deletes that key. See
+   * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
    */
   metadata: Record<string, string | null>;
   /** If true, replace the field's entire metadata map instead of merging. */

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.10",
+  "version": "0.38.0-beta.11",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.10",
+	"version": "0.38.0-beta.11",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.38.0-beta.10",
+  "version": "0.38.0-beta.11",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.38.0-beta.10",
+      "version": "0.38.0-beta.11",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.38.0-beta.10",
+  "version": "0.38.0-beta.11",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/__init__.py

@@ -6,7 +6,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union, Any, List, Iterable
+from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
-from .schema import blob, vector, BlobType
+from .schema import blob, vector
 from .job import AsyncJob, Job
 from .functions import (
     FunctionArtifactRequest as FunctionArtifactRequest,
@@ -49,6 +49,19 @@ from .namespace import (
 )
 
 
+if TYPE_CHECKING:
+    from lance.blob import BlobType as BlobType
+
+
+def __getattr__(name: str):
+    if name == "BlobType":
+        from .schema import BlobType
+
+        globals()["BlobType"] = BlobType
+        return BlobType
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
 def _check_s3_bucket_with_dots(
     uri: str, storage_options: Optional[Dict[str, str]]
 ) -> None:

python/python/lancedb/_blob.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
 import pyarrow as pa
 
 from .expr import Expr
-from .schema import blob_v2_column_paths
+from .schema import row_addressable_blob_v2_paths
 from .types import BlobMode, QueryProjection, QueryProjectionSpec
 
 if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
     schema: pa.Schema,
     projection: QueryProjection,
 ) -> dict[str, str]:
-    blob_columns = blob_v2_column_paths(schema)
+    blob_columns = row_addressable_blob_v2_paths(schema)
     if not blob_columns:
         return {}
     columns = set(blob_columns)
@@ -140,7 +140,9 @@ def v2_projection_needs_row_id(
 ) -> bool:
     if with_row_id:
         return False
-    return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
+    return projection_includes_blob_column(
+        projection, row_addressable_blob_v2_paths(schema)
+    )
 
 
 def blob_auto_row_id_for_scan(
@@ -270,7 +272,8 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
+                yield name, source if source is not None else expr.to_sql()
         return
     for column in projection:
         if isinstance(column, str):
@@ -280,7 +283,8 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                yield name, expr.to_sql()
+                source = expr._column_name()
+                yield name, source if source is not None else expr.to_sql()
 
 
 def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:

python/python/lancedb/_lancedb.pyi

@@ -87,6 +87,7 @@ class PyExpr:
     def contains(self, substr: "PyExpr") -> "PyExpr": ...
     def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
     def cast(self, data_type: pa.DataType) -> "PyExpr": ...
+    def column_name(self) -> Optional[str]: ...
     def to_sql(self) -> str: ...
 
 def expr_col(name: str) -> PyExpr: ...
@@ -608,6 +609,7 @@ class PyQueryRequest:
     filter: Optional[Union[str, bytes]]
     full_text_search: Optional[FullTextQuery]
     select: Optional[Union[str, List[str]]]
+    select_source_columns: Optional[Dict[str, str]]
     fast_search: Optional[bool]
     with_row_id: Optional[bool]
     use_lsm: Optional[bool]

python/python/lancedb/db.py

@@ -16,6 +16,7 @@ from typing import (
     Iterable,
     List,
     Literal,
+    Mapping,
     Optional,
     Union,
 )
@@ -687,17 +688,35 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("serialize is not supported for this connection type")
 
-    def create_function(self, definition: UdfDefinition) -> FunctionVersion:
+    def create_function(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> FunctionVersion:
         """Register a scalar Python UDF and wait for its immutable version.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         This is the blocking counterpart of :meth:`create_function_async`.
         Local connections raise ``NotImplementedError``.
         """
-        return self.create_function_async(definition).wait()
+        return self.create_function_async(definition, secrets=secrets).wait()
 
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         Submission returns a typed job. The immutable Function version becomes
         available only when :meth:`Job.wait` succeeds. Local connections raise
         ``NotImplementedError``.
@@ -1405,8 +1424,13 @@ class LanceDBConnection(DBConnection):
         return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
-        job = LOOP.run(self._conn.create_function_async(definition))
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
+        job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
         return Job(job)
 
     @override
@@ -2225,17 +2249,24 @@ class AsyncConnection(object):
         return AsyncJob(self._inner.job(job_id))
 
     async def create_function_async(
-        self, definition: UdfDefinition
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
     ) -> AsyncJob[FunctionVersion]:
         """Register a scalar Python UDF through the remote Function catalog.
 
+        ``secrets`` must contain exactly the names declared by
+        ``@udf(secrets=[...])``. Values are sent in the create request and
+        stored server-side in the private execution artifact; returned
+        Function and Job metadata contain only the declared names.
         The returned typed job resolves to the immutable Function version.
         Local connections raise ``NotImplementedError``.
         """
         if not isinstance(definition, UdfDefinition):
             raise TypeError("create_function_async requires a @udf definition")
         inner = await self._inner.create_function_async(
-            definition.registration_request.to_canonical_json()
+            definition._submission_json(secrets)
         )
         return _typed_job(inner, FunctionVersion.from_json)
 

python/python/lancedb/expr.py

@@ -249,6 +249,10 @@ class Expr:
 
     # ── utilities ────────────────────────────────────────────────────────────
 
+    def _column_name(self) -> str | None:
+        """Return the source name when this is a bare column expression."""
+        return self._inner.column_name()
+
     def to_sql(self) -> str:
         """Render the expression as a SQL string (useful for debugging)."""
         return self._inner.to_sql()
@@ -312,7 +316,7 @@ def func(name: str, *args: ExprLike) -> Expr:
     --------
     >>> from lancedb.expr import col, func
     >>> func("lower", col("name"))
-    Expr(lower(name))
+    Expr(lower(`name`))
     """
     inner_args = [_coerce(a)._inner for a in args]
     return Expr(expr_func(name, inner_args))

python/python/lancedb/functions.py

@@ -4,7 +4,7 @@
 """Canonical Function values exchanged with LanceDB Enterprise services.
 
 These immutable models contain client/wire state only. Catalog persistence,
-environment bake, and execution are owned by Sophon.
+environment bake, secret resolution, and execution are owned by Sophon.
 ``RefreshColumnResult`` is also the backend-neutral result of a local
 expression-backed refresh job.
 """
@@ -229,7 +229,7 @@ class PythonEnvironmentSpec(_RemoteValue):
 
 
 class PythonRuntimeSpec(_RemoteValue):
-    """Remote runtime definition with environment values.
+    """Remote runtime definition with non-secret environment values.
 
     V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
     their unknown payload fields are intentionally not retained by the client.
@@ -268,6 +268,7 @@ class FunctionVersion(_RemoteValue):
     runtime: PythonRuntimeSpec
     runtime_digest: str
     environment_digest: str
+    required_secrets: tuple[str, ...] = ()
     created_at: str
 
     def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -329,12 +330,17 @@ class FunctionVersion(_RemoteValue):
 
 
 class FunctionRegistrationRequest(_RemoteValue):
-    """Stable remote registration envelope produced by :func:`udf`."""
+    """Stable remote registration envelope produced by :func:`udf`.
+
+    Only secret names are represented. Secret values are supplied separately
+    when the definition is submitted and are not part of this durable value.
+    """
 
     name: str
     artifact: FunctionArtifactRequest
     signature: FunctionSignature
     runtime: PythonRuntimeSpec
+    required_secrets: tuple[str, ...] = ()
 
 
 class FunctionVersionRef(_OpenRemoteValue):
@@ -479,6 +485,27 @@ class RefreshColumnResult(_RemoteValue):
 
 
 _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
+_SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+# Keep this byte limit aligned with Sophon's MAX_FUNCTION_SECRET_VALUE_BYTES.
+_MAX_FUNCTION_SECRET_VALUE_BYTES = 64 * 1024
+_MAX_FUNCTION_SECRET_VALUES_BYTES = 512 * 1024
+
+
+def _validate_secret_value(name: str, value: Any) -> str:
+    """Validate one secret value before building the create request."""
+    if not isinstance(value, str):
+        raise TypeError(f"Function secret {name!r} value must be a string")
+    if not value:
+        raise ValueError(f"Function secret {name!r} value must be non-empty")
+    if "\0" in value:
+        raise ValueError(f"Function secret {name!r} value must not contain NUL")
+    value_bytes = len(value.encode("utf-8"))
+    if value_bytes > _MAX_FUNCTION_SECRET_VALUE_BYTES:
+        raise ValueError(
+            f"Function secret {name!r} value exceeds the "
+            f"{_MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
+        )
+    return value
 
 
 _GRAMMAR_PRIMITIVES = (
@@ -909,6 +936,7 @@ class UdfDefinition:
         output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
         pip: tuple[str, ...],
         env: Mapping[str, str],
+        secrets: tuple[str, ...],
         python_version: Optional[str],
         conda: tuple[str, ...] = (),
         conda_channels: tuple[str, ...] = (),
@@ -935,6 +963,17 @@ class UdfDefinition:
             for key, value in environment.items()
         ):
             raise TypeError("Function env keys and values must be strings")
+        required_secrets = tuple(sorted(set(secrets)))
+        invalid_secrets = [
+            secret for secret in required_secrets if not _SECRET_NAME.fullmatch(secret)
+        ]
+        if invalid_secrets:
+            raise ValueError(f"invalid Function secret names: {invalid_secrets!r}")
+        overlap = set(environment) & set(required_secrets)
+        if overlap:
+            raise ValueError(
+                f"Function env and secret names must be disjoint: {sorted(overlap)!r}"
+            )
         signature = _infer_signature(function, input_schema, output_schema)
         source = _package_source(function)
         digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
@@ -963,14 +1002,65 @@ class UdfDefinition:
             ),
             signature=signature,
             runtime=runtime,
+            required_secrets=required_secrets,
         )
         functools.update_wrapper(self, function)
 
     @property
     def registration_request(self) -> FunctionRegistrationRequest:
-        """The immutable request sent by ``create_function_async``."""
+        """The immutable, value-free client model for a Function submission."""
         return self._request
 
+    def _submission_json(self, secrets: Optional[Mapping[str, str]]) -> str:
+        """Build one registration submission without retaining values on self."""
+        if secrets is None:
+            secret_values: Mapping[str, str] = {}
+        elif not isinstance(secrets, Mapping):
+            raise TypeError("Function secrets must be a mapping of names to strings")
+        else:
+            secret_values = secrets
+
+        if any(not isinstance(name, str) for name in secret_values):
+            raise TypeError("Function secret names must be strings")
+        expected = set(self._request.required_secrets)
+        provided = set(secret_values)
+        if provided != expected:
+            missing = sorted(expected - provided)
+            unexpected = sorted(provided - expected)
+            details = []
+            if missing:
+                details.append(f"missing: {missing!r}")
+            if unexpected:
+                details.append(f"unexpected: {unexpected!r}")
+            raise ValueError(
+                "Function secret values must exactly match the declared secrets ("
+                + "; ".join(details)
+                + ")"
+            )
+
+        canonical_values = {}
+        total_bytes = 0
+        for name in sorted(secret_values):
+            value = _validate_secret_value(name, secret_values[name])
+            total_bytes += len(value.encode("utf-8"))
+            if total_bytes > _MAX_FUNCTION_SECRET_VALUES_BYTES:
+                raise ValueError(
+                    "Function secret values exceed the "
+                    f"{_MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
+                )
+            canonical_values[name] = value
+
+        submission = self._request._known_dict()
+        if canonical_values:
+            submission["secret_values"] = canonical_values
+        return json.dumps(
+            submission,
+            ensure_ascii=False,
+            allow_nan=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+
     def __call__(self, *args, **kwargs):
         return self._function(*args, **kwargs)
 
@@ -988,6 +1078,7 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
@@ -1002,6 +1093,7 @@ def udf(
     output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
     pip: tuple[str, ...] | list[str] = (),
     env: Optional[Mapping[str, str]] = None,
+    secrets: tuple[str, ...] | list[str] = (),
     python_version: Optional[str] = None,
     conda: tuple[str, ...] | list[str] = (),
     conda_channels: tuple[str, ...] | list[str] = (),
@@ -1032,7 +1124,10 @@ def udf(
     conda_channels : sequence of str, optional
         Conda channels in priority order; requires ``conda``.
     env : mapping of str to str, optional
-        Environment variables included in the Function definition.
+        Non-secret environment variables. Use ``secrets`` for credentials.
+    secrets : sequence of str, optional
+        Names of secrets required by the callable. Supply their values separately
+        to ``create_function`` or ``create_function_async``.
     python_version : str, optional
         Remote Python major/minor version. Defaults to the client version.
 
@@ -1054,11 +1149,15 @@ def udf(
     Examples
     --------
     >>> from lancedb import udf
-    >>> @udf(pip=["numpy==2.2.0"])
+    >>> @udf(pip=["numpy==2.2.0"], secrets=["MODEL_TOKEN"])
     ... def score(value: float) -> float:
     ...     return value * 2
     >>> score(1.5)
     3.0
+    >>> db.create_function(  # doctest: +SKIP
+    ...     score, secrets={"MODEL_TOKEN": "user-secret-value"}
+    ... )
+
     """
 
     def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1069,6 +1168,7 @@ def udf(
             output_schema=output_schema,
             pip=tuple(pip),
             env={} if env is None else env,
+            secrets=tuple(secrets),
             python_version=python_version,
             conda=tuple(conda),
             conda_channels=tuple(conda_channels),

python/python/lancedb/query.py

@@ -167,6 +167,12 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
     return {"columns": projection}
 
 
+def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
+    if req.select_source_columns is not None:
+        return req.select_source_columns
+    return req.select
+
+
 def _scanner_kwargs_for_query(
     query: Query,
     blob_mode: BlobMode,
@@ -2799,15 +2805,16 @@ class AsyncQueryBase(object):
 
         req = self._inner.to_query_request()
         schema = await self._table.schema()
+        projection = _query_request_projection(req)
         self._blob_auto_row_id = blob_auto_row_id_for_scan(
             schema,
-            req.select,
+            projection,
             with_row_id=self._with_row_id,
         )
         if not self._blob_auto_row_id:
             self._blob_paths = ()
             return
-        self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
+        self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
         self._inner.with_row_id()
 
     def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3401,9 +3408,10 @@ class AsyncQuery(AsyncStandardQuery):
         pass in multiple vectors. When multiple vectors are passed in, if the vector
         column is with multivector type, then the vectors will be treated as a single
         query. Or the vectors will be treated as multiple queries, this can be useful
-        if you want to find the nearest vectors to multiple query vectors.
-        This is not expected to be faster than making multiple queries concurrently;
-        it is just a convenience method. If multiple vectors are passed in then
+        if you want to find the nearest vectors to multiple query vectors. Flat
+        searches share one table scan across the query vectors, avoiding the scan
+        and memory amplification of making multiple queries concurrently. If
+        multiple vectors are passed in then
         an additional column `query_index` will be added to the results. This column
         will contain the index of the query vector that the result is nearest to.
         """
@@ -3532,8 +3540,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
 
         Typically, a single vector is passed in as the query. However, you can also
         pass in multiple vectors.  This can be useful if you want to find the nearest
-        vectors to multiple query vectors. This is not expected to be faster than
-        making multiple queries concurrently; it is just a convenience method.
+        vectors to multiple query vectors. Flat searches share one table scan across
+        the query vectors instead of issuing concurrent full scans.
         If multiple vectors are passed in then an additional column `query_index`
         will be added to the results.  This column will contain the index of the
         query vector that the result is nearest to.
@@ -3893,14 +3901,15 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         blob_paths: tuple[str, ...] = ()
         if self._table is not None:
             schema = await self._table.schema()
+            projection = _query_request_projection(req)
             blob_auto_row_id = blob_auto_row_id_for_scan(
                 schema,
-                req.select,
+                projection,
                 with_row_id=self._with_row_id,
             )
             if blob_auto_row_id:
                 blob_paths = tuple(
-                    blob_v2_projection_sources(schema, req.select).keys()
+                    blob_v2_projection_sources(schema, projection).keys()
                 )
         self._blob_auto_row_id = blob_auto_row_id
         self._blob_paths = blob_paths

python/python/lancedb/remote/db.py

@@ -7,7 +7,7 @@ import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union
 from urllib.parse import urlparse
 import warnings
 
@@ -742,8 +742,15 @@ class RemoteDBConnection(DBConnection):
         return Job(self._conn.job(job_id))
 
     @override
-    def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
-        return Job(LOOP.run(self._conn.create_function_async(definition)))
+    def create_function_async(
+        self,
+        definition: UdfDefinition,
+        *,
+        secrets: Optional[Mapping[str, str]] = None,
+    ) -> Job[FunctionVersion]:
+        return Job(
+            LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
+        )
 
     @override
     def get_function(self, name: str, *, version: str) -> FunctionVersion:

python/python/lancedb/remote/table.py

@@ -36,6 +36,7 @@ from lancedb._lancedb import (
     UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
+from lancedb.expr import Expr
 from lancedb.index import (
     FTS,
     BTree,
@@ -863,7 +864,7 @@ class RemoteTable(Table):
 
     def update(
         self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -874,9 +875,11 @@ class RemoteTable(Table):
 
         Parameters
         ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.

python/python/lancedb/schema.py

@@ -4,30 +4,34 @@
 
 """Schema helpers for Lance blob columns."""
 
+import importlib
+from typing import TYPE_CHECKING
+
 import pyarrow as pa
+import pyarrow.ipc
+
+if TYPE_CHECKING:
+    from lance.blob import BlobType as BlobType
 
 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
+_BLOB_V2_STORAGE_TYPE = pa.struct(
+    [
+        pa.field("data", pa.large_binary(), nullable=True),
+        pa.field("uri", pa.utf8(), nullable=True),
+        pa.field("position", pa.uint64(), nullable=True),
+        pa.field("size", pa.uint64(), nullable=True),
+    ]
+)
+_resolved_blob_type = None
 
 
-class BlobType(pa.ExtensionType):
-    """PyArrow extension type for a Lance blob v2 column.
-
-    Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
-    for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
-    """
+class _FallbackBlobType(pa.ExtensionType):
+    """lance.blob.v2 extension type used when pylance is not installed."""
 
     def __init__(self) -> None:
-        storage_type = pa.struct(
-            [
-                pa.field("data", pa.large_binary(), nullable=True),
-                pa.field("uri", pa.utf8(), nullable=True),
-                pa.field("position", pa.uint64(), nullable=True),
-                pa.field("size", pa.uint64(), nullable=True),
-            ]
-        )
-        super().__init__(storage_type, _BLOB_EXTENSION_NAME)
+        pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
 
     def __arrow_ext_serialize__(self) -> bytes:
         return b""
@@ -35,23 +39,16 @@ class BlobType(pa.ExtensionType):
     @classmethod
     def __arrow_ext_deserialize__(
         cls, storage_type: pa.DataType, serialized: bytes
-    ) -> "BlobType":
+    ) -> "_FallbackBlobType":
         return cls()
 
     def __reduce__(self):
-        # Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
         return type(self).__arrow_ext_deserialize__, (
             self.storage_type,
             self.__arrow_ext_serialize__(),
         )
 
 
-try:
-    pa.register_extension_type(BlobType())  # type: ignore[arg-type]
-except pa.ArrowKeyError:
-    pass
-
-
 def _metadata_value(metadata: dict, key: str):
     return metadata.get(key.encode()) or metadata.get(key)
 
@@ -92,43 +89,105 @@ def is_blob_like_field(field: pa.Field) -> bool:
     return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
 
 
-def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
-    paths: list[str] = []
+def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
+    """Walk the schema and return (path, has_list_ancestor) for each blob field."""
+    paths: list[tuple[str, bool]] = []
 
-    def walk(fields, prefix: str) -> None:
+    def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
         for field in fields:
             path = f"{prefix}.{field.name}" if prefix else field.name
             if is_blob(field):
-                paths.append(path)
+                paths.append((path, has_list_ancestor))
             elif pa.types.is_struct(field.type):
-                walk(field.type, path)
+                walk(field.type, path, has_list_ancestor)
             elif (
                 pa.types.is_list(field.type)
                 or pa.types.is_large_list(field.type)
                 or pa.types.is_fixed_size_list(field.type)
             ):
-                walk([field.type.value_field], path)
+                walk([field.type.value_field], path, True)
 
-    walk(schema, "")
+    walk(schema, "", False)
     return paths
 
 
 def blob_column_paths(schema: pa.Schema) -> list[str]:
     """Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
-    return _collect_blob_paths(schema, is_blob_like_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
 
 
 def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
-    return _collect_blob_paths(schema, is_blob_v2_field)
+    return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
+
+
+def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
+    """Blob v2 paths with one blob addressable by table row id.
+
+    ``fetch_blobs`` and the descriptor row-id ride-along address one blob per
+    row, so a blob inside a list container has no row-id slot and no fetch
+    path. Those columns still store and query as raw descriptors.
+    """
+    return [
+        path
+        for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
+        if not has_list_ancestor
+    ]
 
 
 def schema_has_blob_field(schema: pa.Schema) -> bool:
     return bool(blob_column_paths(schema))
 
 
+def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
+    """Return the type Arrow reconstructs for this extension name."""
+    schema = pa.schema([pa.field("value", extension_type)])
+    restored = pa.ipc.read_schema(schema.serialize())
+    return restored.field("value").type
+
+
+def _resolve_blob_type():
+    """Return the BlobType class this process should use.
+
+    pylance's class when it owns the lance.blob.v2 registry entry,
+    otherwise LanceDB's fallback. A different registered class is an error.
+    """
+    global _resolved_blob_type
+    if _resolved_blob_type is not None:
+        return _resolved_blob_type
+    try:
+        blob_module = importlib.import_module("lance.blob")
+    except ModuleNotFoundError as err:
+        if err.name not in ("lance", "lance.blob"):
+            raise
+    else:
+        blob_type = getattr(blob_module, "BlobType", None)
+        if blob_type is not None:
+            registered_type = _deserialize_registered_type(blob_type())
+            if type(registered_type) is not blob_type:
+                registered_cls = type(registered_type)
+                raise ValueError(
+                    "lance.blob.v2 is already registered by "
+                    f"{registered_cls.__module__}.{registered_cls.__qualname__}"
+                )
+            _resolved_blob_type = blob_type
+            return blob_type
+    try:
+        pa.register_extension_type(_FallbackBlobType())  # type: ignore[arg-type]
+    except pa.ArrowKeyError as err:
+        raise ValueError(
+            "lance.blob.v2 is already registered by another extension class"
+        ) from err
+    _resolved_blob_type = _FallbackBlobType
+    return _resolved_blob_type
+
+
 def blob(name: str, nullable: bool = True) -> pa.Field:
-    """Create a Lance blob v2 column field."""
-    return pa.field(name, BlobType(), nullable=nullable)
+    """Create a Lance blob v2 column field.
+
+    When pylance is installed this is ``lance.blob.BlobType``.
+    """
+    blob_type = _resolve_blob_type()
+    return pa.field(name, blob_type(), nullable=nullable)
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -155,3 +214,11 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
     ... ])
     """
     return pa.list_(value_type, dimension)
+
+
+def __getattr__(name: str):
+    if name == "BlobType":
+        blob_type = _resolve_blob_type()
+        globals()["BlobType"] = blob_type
+        return blob_type
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

python/python/lancedb/table.py

@@ -104,7 +104,12 @@ from .util import (
     value_to_sql,
 )
 from .index import lang_mapping
-from .schema import blob_v2_column_paths, schema_has_blob_field
+from .schema import (
+    blob_v2_column_paths,
+    is_blob_v2_field,
+    row_addressable_blob_v2_paths,
+    schema_has_blob_field,
+)
 
 
 def _should_push_down_query_table(
@@ -426,6 +431,7 @@ def _cast_to_target_schema(
 
     def gen():
         for batch in reader:
+            batch = _coerce_blob_write_columns(batch, reordered_schema)
             # Table but not RecordBatch has cast.
             cast_batches = (
                 pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -438,6 +444,166 @@ def _cast_to_target_schema(
     return pa.RecordBatchReader.from_batches(reordered_schema, gen())
 
 
+def _coerce_blob_write_columns(
+    batch: pa.RecordBatch, target_schema: pa.Schema
+) -> pa.RecordBatch:
+    """Materialize blob storage structs before the stream leaves Python.
+
+    merge_insert requires its source reader to already match the table's
+    physical schema. Unlike add and insert, it does not pass through
+    LanceDB's Rust blob coercion, so preserving binary input here would
+    reach Lance as binary and fail the schema check.
+    """
+    columns = []
+    fields = []
+    changed = False
+    for field, column in zip(batch.schema, batch.columns):
+        target_field = target_schema.field(field.name)
+        coerced = _coerce_blob_value(column, target_field)
+        if coerced is not column:
+            column = coerced
+            field = pa.field(
+                field.name,
+                coerced.type,
+                field.nullable,
+                target_field.metadata,
+            )
+            changed = True
+        columns.append(column)
+        fields.append(field)
+    if not changed:
+        return batch
+    return pa.RecordBatch.from_arrays(
+        columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
+    )
+
+
+def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
+    if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
+        return _coerce_value_to_blob(column, target_field)
+
+    target_type = target_field.type
+    if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
+        children = []
+        fields = []
+        changed = False
+        for source_field in column.type:
+            source_column = column.field(source_field.name)
+            nested_target = next(
+                (field for field in target_type if field.name == source_field.name),
+                None,
+            )
+            if nested_target is None:
+                children.append(source_column)
+                fields.append(source_field)
+                continue
+            coerced = _coerce_blob_value(source_column, nested_target)
+            if coerced is not source_column:
+                changed = True
+            child_array, child_type = _physical_array_and_type(coerced)
+            children.append(child_array)
+            fields.append(
+                pa.field(
+                    source_field.name,
+                    child_type,
+                    source_field.nullable,
+                    nested_target.metadata,
+                )
+            )
+        if not changed:
+            return column
+        return pa.StructArray.from_arrays(
+            children,
+            fields=fields,
+            mask=column.is_null() if column.null_count else None,
+        )
+
+    if _is_list_like(target_type) and _is_list_like(column.type):
+        return _coerce_blob_list_values(column, target_type.value_field)
+
+    return column
+
+
+def _coerce_blob_list_values(
+    column: pa.Array, target_value_field: pa.Field
+) -> pa.Array:
+    """Coerce blob values inside a list column, preserving offsets and nulls.
+
+    Works on the raw child values window instead of ``pc.list_flatten`` because
+    flatten drops values spanned by null slots, which would misalign offsets.
+    """
+    mask = column.is_null() if column.null_count else None
+    if pa.types.is_fixed_size_list(column.type):
+        list_size = column.type.list_size
+        values = column.values.slice(column.offset * list_size, len(column) * list_size)
+        coerced = _coerce_blob_value(values, target_value_field)
+        if coerced is values:
+            return column
+        physical_values, _ = _physical_array_and_type(coerced)
+        return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
+    offsets = column.offsets
+    first_offset = offsets[0].as_py()
+    values = column.values.slice(
+        first_offset,
+        offsets[-1].as_py() - first_offset,
+    )
+    coerced = _coerce_blob_value(values, target_value_field)
+    if coerced is values:
+        return column
+    physical_values, _ = _physical_array_and_type(coerced)
+    if first_offset:
+        offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
+    if pa.types.is_large_list(column.type):
+        return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
+    return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
+
+
+def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
+    if pa.types.is_null(values.type):
+        data = pa.nulls(len(values), type=pa.large_binary())
+    elif pa.types.is_large_binary(values.type):
+        data = values
+    else:
+        data = values.cast(pa.large_binary())
+    length = len(values)
+    storage_type = target_field.type
+    if isinstance(storage_type, pa.ExtensionType):
+        storage_type = storage_type.storage_type
+    storage_fields = list(storage_type)
+    children = []
+    for storage_field in storage_fields:
+        if storage_field.name == "data":
+            children.append(data)
+        else:
+            children.append(pa.nulls(length, type=storage_field.type))
+    storage = pa.StructArray.from_arrays(
+        children,
+        fields=storage_fields,
+        mask=values.is_null() if values.null_count else None,
+    )
+    if isinstance(target_field.type, pa.ExtensionType):
+        return pa.ExtensionArray.from_storage(target_field.type, storage)
+    return storage
+
+
+def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
+    if isinstance(array.type, pa.ExtensionType):
+        return array.storage, array.type.storage_type
+    return array, array.type
+
+
+def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
+    return _is_binary_like(data_type) or pa.types.is_null(data_type)
+
+
+def _is_binary_like(data_type: pa.DataType) -> bool:
+    return (
+        pa.types.is_binary(data_type)
+        or pa.types.is_large_binary(data_type)
+        or pa.types.is_binary_view(data_type)
+    )
+
+
 def _field_extension_name(field: pa.Field) -> Optional[str]:
     extension_name = getattr(field.type, "extension_name", None)
     if extension_name is not None:
@@ -464,63 +630,71 @@ def _align_field_types(
         target_field = next((f for f in target_fields if f.name == field.name), None)
         if target_field is None:
             raise ValueError(f"Field '{field.name}' not found in target schema")
-        # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
-        # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
-        # input to that storage type here merely relabels the raw JSON bytes as
-        # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
-        if (
-            _field_extension_name(field) == "arrow.json"
-            and _field_extension_name(target_field) == "lance.json"
-        ):
-            new_fields.append(field)
-            continue
-        if pa.types.is_struct(target_field.type):
-            if pa.types.is_struct(field.type):
-                new_type = pa.struct(
-                    _align_field_types(
-                        field.type.fields,
-                        target_field.type.fields,
-                    )
+        new_fields.append(_align_field(field, target_field))
+    return new_fields
+
+
+def _align_list_value_field(
+    value_field: pa.Field, target_value_field: pa.Field
+) -> pa.Field:
+    # A list has exactly one child, so the inferred child name ("item") aligns
+    # positionally and adopts the table's child name; pa.Table.cast renames it.
+    return _align_field(value_field, target_value_field).with_name(
+        target_value_field.name
+    )
+
+
+def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
+    # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
+    # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
+    # input to that storage type here merely relabels the raw JSON bytes as
+    # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
+    if (
+        _field_extension_name(field) == "arrow.json"
+        and _field_extension_name(target_field) == "lance.json"
+    ):
+        return field
+    if pa.types.is_struct(target_field.type):
+        if pa.types.is_struct(field.type):
+            new_type = pa.struct(
+                _align_field_types(
+                    field.type.fields,
+                    target_field.type.fields,
                 )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.list_(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0]
-                )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_large_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.large_list(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0]
-                )
-            else:
-                new_type = target_field.type
-        elif pa.types.is_fixed_size_list(target_field.type):
-            if _is_list_like(field.type):
-                new_type = pa.list_(
-                    _align_field_types(
-                        [field.type.value_field],
-                        [target_field.type.value_field],
-                    )[0],
-                    target_field.type.list_size,
-                )
-            else:
-                new_type = target_field.type
+            )
         else:
             new_type = target_field.type
-        new_fields.append(
-            pa.field(field.name, new_type, field.nullable, target_field.metadata)
-        )
-    return new_fields
+    elif pa.types.is_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.list_(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                )
+            )
+        else:
+            new_type = target_field.type
+    elif pa.types.is_large_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.large_list(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                )
+            )
+        else:
+            new_type = target_field.type
+    elif pa.types.is_fixed_size_list(target_field.type):
+        if _is_list_like(field.type):
+            new_type = pa.list_(
+                _align_list_value_field(
+                    field.type.value_field, target_field.type.value_field
+                ),
+                target_field.type.list_size,
+            )
+        else:
+            new_type = target_field.type
+    else:
+        new_type = target_field.type
+    return pa.field(field.name, new_type, field.nullable, target_field.metadata)
 
 
 def _infer_subschema(
@@ -589,7 +763,7 @@ def sanitize_create_table(
         schema = data.schema
     else:
         if schema is not None:
-            data = pa.Table.from_pylist([], schema)
+            data = pa.Table.from_batches([], schema=schema)
     if schema is None:
         if data is None:
             raise ValueError("Either data or schema must be provided")
@@ -1744,7 +1918,7 @@ class Table(ABC):
     @abstractmethod
     def update(
         self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -1759,9 +1933,11 @@ class Table(ABC):
 
         Parameters
         ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -1779,6 +1955,7 @@ class Table(ABC):
         Examples
         --------
         >>> import lancedb
+        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -1788,7 +1965,7 @@ class Table(ABC):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -2127,12 +2304,25 @@ class Table(ABC):
         ----------
         updates : dict
             One or more dicts, each with:
+
             - "path": str — dot-path to the field (e.g. "embedding" or "a.b.c").
             - "metadata": dict[str, str | None] — keys to set; a value of ``None``
               deletes that key.
             - "replace": bool, optional — replace the field's whole metadata map
               instead of merging (default False).
 
+            The following keys are treated specially, by convention, and should
+            be used when appropriate:
+
+            - "lancedb:description": for a human-readable description of a field.
+            - ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
+                suffix names the tag category; e.g. "lancedb:tag:model": "clip".
+            - "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
+                and "feature_v2" might be in the same logical column.
+            - "lancedb:status" for status options ("production", "candidate",
+                "deprecated", "archived") to designate the current life cycle
+                state of this column.
+
         Returns
         -------
         UpdateFieldMetadataResult
@@ -2682,7 +2872,7 @@ class LanceTable(Table):
             arrow_tbl = self.to_arrow()
             if blob_mode == "descriptions":
                 arrow_tbl = strip_auto_row_ids(
-                    arrow_tbl, blob_v2_column_paths(self.schema)
+                    arrow_tbl, row_addressable_blob_v2_paths(self.schema)
                 )
             return arrow_tbl.to_pandas(**kwargs)
 
@@ -3828,7 +4018,7 @@ class LanceTable(Table):
 
     def update(
         self,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -3839,9 +4029,11 @@ class LanceTable(Table):
 
         Parameters
         ----------
-        where: str, optional
-            The SQL where clause to use when updating rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
+            error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -3859,6 +4051,7 @@ class LanceTable(Table):
         Examples
         --------
         >>> import lancedb
+        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -3868,7 +4061,7 @@ class LanceTable(Table):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
+        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -5083,7 +5276,9 @@ class AsyncTable:
         if blob_mode == "descriptions" or not schema_has_blob_field(schema):
             arrow_tbl = await self.to_arrow()
             if blob_mode == "descriptions":
-                arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
+                arrow_tbl = strip_auto_row_ids(
+                    arrow_tbl, row_addressable_blob_v2_paths(schema)
+                )
             return arrow_tbl.to_pandas(**kwargs)
 
         if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -5982,7 +6177,7 @@ class AsyncTable:
         self,
         updates: Optional[Dict[str, Any]] = None,
         *,
-        where: Optional[str] = None,
+        where: Optional[Union[str, Expr]] = None,
         updates_sql: Optional[Dict[str, str]] = None,
     ) -> UpdateResult:
         """
@@ -5997,9 +6192,11 @@ class AsyncTable:
             The updates to apply.  The keys should be the name of the column to
             update.  The values should be the new values to assign.  This is
             required unless updates_sql is supplied.
-        where: str, optional
-            An SQL filter that controls which rows are updated. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'.  Only rows that satisfy this filter will be udpated.
+        where: str or [Expr][lancedb.expr.Expr], optional
+            The filter condition. Can be a SQL string or a type-safe
+            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
+            [lit][lancedb.expr.lit]. Only rows that satisfy this filter will
+            be updated.
         updates_sql: dict, optional
             The updates to apply, expressed as SQL expression strings.  The keys should
             be column names. The values should be SQL expressions.  These can be SQL
@@ -6017,13 +6214,14 @@ class AsyncTable:
         --------
         >>> import asyncio
         >>> import lancedb
+        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> async def demo_update():
         ...     data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
         ...     db = await lancedb.connect_async("./.lancedb")
         ...     table = await db.create_table("my_table", data)
         ...     # x is [1, 2], vector is [[1, 2], [3, 4]]
-        ...     await table.update({"vector": [10, 10]}, where="x = 2")
+        ...     await table.update({"vector": [10, 10]}, where=col("x") == 2)
         ...     # x is [1, 2], vector is [[1, 2], [10, 10]]
         ...     await table.update(updates_sql={"x": "x + 1"})
         ...     # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6037,7 +6235,8 @@ class AsyncTable:
         if updates is not None:
             updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
 
-        return await self._inner.update(updates_sql, where)
+        predicate = where.to_sql() if isinstance(where, Expr) else where
+        return await self._inner.update(updates_sql, predicate)
 
     async def add_columns(
         self,

python/python/tests/docs/test_basic.py

@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
     tbl.create_index(num_sub_vectors=1)
     # --8<-- [end:create_index]
     # --8<-- [start:delete_rows]
-    tbl.delete('item = "fizz"')
+    tbl.delete("item = 'fizz'")
     # --8<-- [end:delete_rows]
     # --8<-- [start:drop_table]
     db.drop_table("my_table")
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
     await tbl.create_index("vector")
     # --8<-- [end:create_index_async]
     # --8<-- [start:delete_rows_async]
-    await tbl.delete('item = "fizz"')
+    await tbl.delete("item = 'fizz'")
     # --8<-- [end:delete_rows_async]
     # --8<-- [start:drop_table_async]
     await db.drop_table("my_table_async")

python/python/tests/docs/test_guide_tables.py

@@ -266,7 +266,7 @@ def test_table():
     tbl.add(pydantic_model_items)
     # --8<-- [end:add_table_from_pydantic]
     # --8<-- [start:delete_row]
-    tbl.delete('item = "fizz"')
+    tbl.delete("item = 'fizz'")
     # --8<-- [end:delete_row]
     # --8<-- [start:delete_specific_row]
     data = [
@@ -538,7 +538,7 @@ async def test_table_async():
     await async_tbl.add(pydantic_model_items)
     # --8<-- [end:add_table_async_from_pydantic]
     # --8<-- [start:delete_row_async]
-    await async_tbl.delete('item = "fizz"')
+    await async_tbl.delete("item = 'fizz'")
     # --8<-- [end:delete_row_async]
     # --8<-- [start:delete_specific_row_async]
     data = [

python/python/tests/test_blob.py

@@ -2,17 +2,41 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import io
+import subprocess
+import sys
+import textwrap
 
+import lance
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
+from lance.blob import BlobType as LanceBlobType
 
 import lancedb
-from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
+from lancedb._blob import (
+    blob_v2_projection_sources,
+    read_row_ids_from_hits,
+    stash_auto_row_ids,
+)
+from lancedb.expr import col
 from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths
 
 
+_HIDE_LANCE_BLOB = """\
+import importlib.abc
+import sys
+
+class _MissingLanceBlob(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path, target=None):
+        if fullname == "lance.blob" or fullname.startswith("lance.blob."):
+            raise ModuleNotFoundError(fullname, name="lance.blob")
+
+sys.modules.pop("lance.blob", None)
+sys.meta_path.insert(0, _MissingLanceBlob())
+"""
+
+
 def _blob_table(name, rows):
     db = lancedb.connect("memory:///")
     schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -46,6 +70,181 @@ def test_blob_factory_declares_v2_field():
     field = lancedb.blob("image")
     assert isinstance(field.type, pa.ExtensionType)
     assert field.type.extension_name == "lance.blob.v2"
+    assert lancedb.BlobType is LanceBlobType
+    assert type(field.type) is LanceBlobType
+
+
+def test_blob_type_works_without_pylance():
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        """\
+        import lancedb
+        import pyarrow as pa
+
+        field = lancedb.blob("image")
+        if not isinstance(field.type, pa.ExtensionType):
+            raise SystemExit("expected an extension type")
+        if field.type.extension_name != "lance.blob.v2":
+            raise SystemExit(field.type.extension_name)
+        if lancedb.BlobType is not type(field.type):
+            raise SystemExit("BlobType is not the field type class")
+        if lancedb.BlobType.__module__ != "lancedb.schema":
+            raise SystemExit(lancedb.BlobType.__module__)
+
+        db = lancedb.connect("memory:///")
+        table = db.create_table(
+            "images",
+            schema=pa.schema([pa.field("id", pa.int64()), field]),
+        )
+        table.add([{"id": 1, "image": b"hello"}])
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"merge_insert rows updated={result.num_updated_rows} "
+                f"inserted={result.num_inserted_rows}"
+            )
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_resolves_pylance_type_without_eager_import():
+    script = textwrap.dedent(
+        """\
+        import sys
+        import lancedb
+
+        if "lance.blob" in sys.modules:
+            raise SystemExit("import lancedb imported lance.blob")
+        field = lancedb.blob("image")
+        from lance.blob import BlobType
+
+        if type(field.type) is not BlobType:
+            raise SystemExit(f"{type(field.type)} is not {BlobType}")
+        import lance
+
+        image = lance.blob_array([b"x"])
+        if type(image.type) is not BlobType:
+            raise SystemExit("blob_array used a different class")
+        if type(image.type) is not type(field.type):
+            raise SystemExit("field and array classes differ")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_fallback_fails_if_name_already_registered():
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        """\
+        import pyarrow as pa
+
+        class OtherBlobType(pa.ExtensionType):
+            def __init__(self):
+                super().__init__(
+                    pa.struct([pa.field("data", pa.large_binary())]),
+                    "lance.blob.v2",
+                )
+
+            def __arrow_ext_serialize__(self):
+                return b""
+
+            @classmethod
+            def __arrow_ext_deserialize__(cls, storage_type, serialized):
+                return cls()
+
+        pa.register_extension_type(OtherBlobType())
+        import lancedb
+
+        try:
+            lancedb.blob("image")
+        except ValueError as err:
+            if "already registered" not in str(err):
+                raise SystemExit(err)
+        else:
+            raise SystemExit("expected ValueError")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_blob_type_rejects_competing_registration_with_pylance():
+    script = textwrap.dedent(
+        """\
+        import pyarrow as pa
+        import pyarrow.ipc
+
+        class OtherBlobType(pa.ExtensionType):
+            def __init__(self):
+                super().__init__(
+                    pa.struct(
+                        [
+                            pa.field("data", pa.large_binary()),
+                            pa.field("uri", pa.utf8()),
+                            pa.field("position", pa.uint64()),
+                            pa.field("size", pa.uint64()),
+                        ]
+                    ),
+                    "lance.blob.v2",
+                )
+
+            def __arrow_ext_serialize__(self):
+                return b""
+
+            @classmethod
+            def __arrow_ext_deserialize__(cls, storage_type, serialized):
+                return cls()
+
+        pa.register_extension_type(OtherBlobType())
+
+        from lance.blob import BlobType
+
+        if BlobType is OtherBlobType:
+            raise SystemExit("pylance BlobType was replaced")
+        schema = pa.schema([pa.field("value", BlobType())])
+        restored = pa.ipc.read_schema(schema.serialize())
+        if type(restored.field("value").type) is not OtherBlobType:
+            raise SystemExit(type(restored.field("value").type))
+
+        import lancedb
+
+        try:
+            lancedb.blob("image")
+        except ValueError as err:
+            if "__main__.OtherBlobType" not in str(err):
+                raise SystemExit(err)
+        else:
+            raise SystemExit("expected ValueError")
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
 
 
 def test_blob_v2_column_paths_include_list_children():
@@ -70,6 +269,14 @@ def test_blob_v2_column_paths_include_list_children():
     ]
 
 
+def test_blob_v2_projection_sources_use_typed_column_name():
+    schema = pa.schema([lancedb.blob("blob")])
+
+    assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
+        "blob_alias": "blob"
+    }
+
+
 def _legacy_v1_table(name):
     db = lancedb.connect("memory:///")
     schema = pa.schema(
@@ -166,6 +373,20 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
     assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
 
 
+@pytest.mark.asyncio
+async def test_async_typed_blob_projection_preserves_source_column():
+    db = await lancedb.connect_async("memory:///typed_blob_projection")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
+    table = await db.create_table("typed_blob_projection", schema=schema)
+    await table.add([{"id": 1, "blob": b"alpha"}])
+
+    hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
+
+    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
+    blobs = await table.fetch_blobs("blob", hits)
+    assert blobs.to_pylist() == [b"alpha"]
+
+
 def test_fetch_blobs_round_trip():
     table = _blob_table(
         "round_trip",
@@ -176,6 +397,292 @@ def test_fetch_blobs_round_trip():
     assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
 
 
+def test_merge_insert_writes_python_bytes():
+    table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
+    )
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [b"updated", b"inserted"]
+
+
+def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"hello"}])
+
+    script = textwrap.dedent(
+        f"""\
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(
+                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
+            )
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"hello"}])
+
+    script = _HIDE_LANCE_BLOB + textwrap.dedent(
+        f"""\
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(f"expected StructType, got {{type(image_type)}}")
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(
+                [{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
+            )
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("images", schema=schema)
+    table.add([{"id": 1, "image": b"before"}])
+
+    script = textwrap.dedent(
+        f"""\
+        import pyarrow as pa
+        import lancedb
+
+        db = lancedb.connect({str(tmp_path)!r})
+        table = db.open_table("images")
+        image_type = table.schema.field("image").type
+        if type(image_type).__name__ != "StructType":
+            raise SystemExit(
+                f"expected StructType before lance import, got {{type(image_type)}}"
+            )
+
+        import lance
+
+        updates = pa.Table.from_arrays(
+            [
+                pa.array([1, 2], type=pa.int64()),
+                lance.blob_array([b"updated", b"inserted"]),
+            ],
+            names=["id", "image"],
+        )
+        result = (
+            table.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(updates)
+        )
+        if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
+            raise SystemExit(
+                f"rows updated={{result.num_updated_rows}} "
+                f"inserted={{result.num_inserted_rows}}"
+            )
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
+        blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+        if blobs.to_pylist() != [b"updated", b"inserted"]:
+            raise SystemExit(blobs.to_pylist())
+        """
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_add_all_null_blob_column():
+    db = lancedb.connect("memory:///")
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("all_null", schema=schema)
+    table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [None, None]
+
+
+def test_create_table_nested_blob_schema_without_rows():
+    db = lancedb.connect("memory:///")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("info", pa.struct([lancedb.blob("blob")])),
+            pa.field("images", pa.list_(lancedb.blob("image"))),
+        ]
+    )
+    table = db.create_table("nested_empty", schema=schema)
+    assert table.count_rows() == 0
+
+
+def test_merge_insert_nested_blob_dicts():
+    db = lancedb.connect("memory:///")
+    info = pa.StructArray.from_arrays(
+        [
+            pa.array(["first"], type=pa.string()),
+            _blob_array("blob", [b"before"]),
+        ],
+        names=["name", "blob"],
+    )
+    data = pa.Table.from_arrays(
+        [pa.array([1], type=pa.int64()), info],
+        names=["id", "info"],
+    )
+    table = db.create_table("nested_merge", data=data)
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
+    )
+    assert result.num_updated_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("info.blob", [by_id[1]])
+    assert blobs.to_pylist() == [b"after"]
+
+
+def _list_blob_table(name):
+    db = lancedb.connect("memory:///")
+    blob_field = lancedb.blob("image")
+    images = pa.ListArray.from_arrays(
+        pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
+    )
+    data = pa.Table.from_arrays(
+        [pa.array([1], type=pa.int64()), images],
+        schema=pa.schema(
+            [pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
+        ),
+    )
+    return db.create_table(name, data=data)
+
+
+def test_merge_insert_list_blob_dicts():
+    table = _list_blob_table("list_merge")
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
+    )
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    hits = table.search().limit(10).to_arrow()
+    sizes = {
+        row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
+        for row in hits.to_pylist()
+    }
+    assert sizes == {1: [3, 3], 2: None}
+
+
+def test_list_blob_column_queries_as_raw_descriptors():
+    table = _list_blob_table("list_query")
+    hits = table.search().limit(10).to_arrow()
+    element = hits.schema.field("images").type.value_type
+    assert pa.types.is_struct(element)
+    assert "_lance_row_id" not in element.names
+    with pytest.raises(ValueError, match="expected struct before segment"):
+        table.fetch_blobs("images.image", [0])
+
+
+def test_row_addressable_paths_exclude_list_children():
+    from lancedb.schema import row_addressable_blob_v2_paths
+
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("info", pa.struct([lancedb.blob("blob")])),
+            pa.field("images", pa.list_(lancedb.blob("image"))),
+        ]
+    )
+    assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
+    assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
+
+
+def test_merge_insert_writes_pylance_blob_array():
+    table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
+    image = lance.blob_array([b"updated", b"inserted"])
+    assert type(image.type) is LanceBlobType
+    assert type(image.type) is type(lancedb.BlobType())
+    updates = pa.Table.from_arrays(
+        [pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
+    )
+
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(updates)
+    )
+
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 1
+    by_id = _row_ids_by_id(table)
+    blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
+    assert blobs.to_pylist() == [b"updated", b"inserted"]
+
+
 def test_fetch_blobs_accepts_query_result():
     table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
     hits = table.search().limit(10).to_arrow()
@@ -403,6 +910,50 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
     assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
 
 
+@pytest.mark.asyncio
+async def test_async_hybrid_typed_blob_projection_preserves_source_column():
+    db = await lancedb.connect_async("memory:///hybrid_typed_blob")
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("text", pa.utf8()),
+            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
+            lancedb.blob("blob"),
+        ]
+    )
+    table = await db.create_table("hybrid_typed_blob", schema=schema)
+    await table.add(
+        [
+            {
+                "id": 1,
+                "text": "hello alpha",
+                "vector": [1.0, 0.0],
+                "blob": b"alpha",
+            },
+            {
+                "id": 2,
+                "text": "hello beta",
+                "vector": [0.9, 0.1],
+                "blob": b"beta",
+            },
+        ]
+    )
+    await table.create_index("text", config=FTS(with_position=False))
+
+    hits = await (
+        table.query()
+        .nearest_to([1.0, 0.0])
+        .nearest_to_text("hello")
+        .select({"blob_alias": col("blob")})
+        .limit(2)
+        .to_arrow()
+    )
+
+    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
+    blobs = await table.fetch_blobs("blob", hits)
+    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
+
+
 def test_blob_file_seek_read_and_read_range():
     payload = _identifiable_payload(1024)
     table = _blob_table("seek_read", [{"id": 1, "image": payload}])

python/python/tests/test_expr.py

@@ -52,7 +52,7 @@ class TestExprConstruction:
     def test_func(self):
         e = func("lower", col("name"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(name)"
+        assert e.to_sql() == "lower(`name`)"
 
     def test_func_unknown_raises(self):
         with pytest.raises(Exception):
@@ -115,7 +115,7 @@ class TestExprOperators:
     def test_and_operator(self):
         e = (col("age") > lit(18)) & (col("status") == lit("active"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "((age > 18) AND (status = 'active'))"
+        assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"
 
     def test_or_operator(self):
         e = (col("a") == lit(1)) | (col("b") == lit(2))
@@ -166,7 +166,7 @@ class TestExprOperators:
     def test_coerce_plain_str(self):
         e = col("name") == "alice"
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(name = 'alice')"
+        assert e.to_sql() == "(`name` = 'alice')"
 
     def test_reflexive_comparisons(self):
         # 10 < col("age") swaps to col("age") > 10
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:
 
     def test_bytes_equality_expr_sql(self):
         e = col("data") == lit(b"\xca\xfe")
-        assert e.to_sql() == "(data = X'CAFE')"
+        assert e.to_sql() == "(`data` = X'CAFE')"
 
     def test_bytes_ne_expr_sql(self):
         e = col("data") != lit(b"\xff")
-        assert e.to_sql() == "(data <> X'FF')"
+        assert e.to_sql() == "(`data` <> X'FF')"
 
     def test_bytes_compound_expr_sql(self):
         e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
-        assert e.to_sql() == "((data = X'01') AND (id > 5))"
+        assert e.to_sql() == "((`data` = X'01') AND (id > 5))"
 
     def test_bytes_in_function_call(self):
         # Regression test: binary literals inside scalar function calls
         # used to fail because DataFusion's unparser does not support Binary
         # scalars.  Now handled via a placeholder-substitution rewrite.
         e = func("contains", col("data"), lit(b"\xff"))
-        assert e.to_sql() == "contains(data, X'FF')"
+        assert e.to_sql() == "contains(`data`, X'FF')"
 
     def test_bytes_in_not(self):
         e = ~(col("data") == lit(b"\xff"))
-        assert e.to_sql() == "NOT (data = X'FF')"
+        assert e.to_sql() == "NOT (`data` = X'FF')"
 
 
 class TestExprStringMethods:
     def test_lower(self):
         e = col("name").lower()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(name)"
+        assert e.to_sql() == "lower(`name`)"
 
     def test_upper(self):
         e = col("name").upper()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "upper(name)"
+        assert e.to_sql() == "upper(`name`)"
 
     def test_contains(self):
         e = col("text").contains(lit("hello"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(text, 'hello')"
+        assert e.to_sql() == "contains(`text`, 'hello')"
 
     def test_contains_with_str_coerce(self):
         e = col("text").contains("hello")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(text, 'hello')"
+        assert e.to_sql() == "contains(`text`, 'hello')"
 
     def test_chained_lower_eq(self):
         e = col("name").lower() == lit("alice")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(lower(name) = 'alice')"
+        assert e.to_sql() == "(lower(`name`) = 'alice')"
 
 
 class TestExprCast:
     def test_cast_string(self):
         e = col("id").cast("string")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(id AS VARCHAR)"
+        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
 
     def test_cast_int32(self):
         e = col("score").cast("int32")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(score AS INTEGER)"
+        assert e.to_sql() == "arrow_cast(score, 'Int32')"
 
     def test_cast_float64(self):
         e = col("val").cast("float64")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(val AS DOUBLE)"
+        assert e.to_sql() == "arrow_cast(val, 'Float64')"
 
     def test_cast_pyarrow_type(self):
         e = col("score").cast(pa.int32())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(score AS INTEGER)"
+        assert e.to_sql() == "arrow_cast(score, 'Int32')"
 
     def test_cast_pyarrow_float64(self):
         e = col("val").cast(pa.float64())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(val AS DOUBLE)"
+        assert e.to_sql() == "arrow_cast(val, 'Float64')"
 
     def test_cast_pyarrow_string(self):
         e = col("id").cast(pa.string())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "CAST(id AS VARCHAR)"
+        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
 
     def test_cast_pyarrow_and_string_equivalent(self):
         # pa.int32() and "int32" should produce equivalent SQL
@@ -597,14 +597,14 @@ class TestExprIsin:
     def test_isin_strs(self):
         assert (
             col("status").isin(["active", "pending"]).to_sql()
-            == "status IN ('active', 'pending')"
+            == "`status` IN ('active', 'pending')"
         )
 
     def test_isin_coerces_and_mixes(self):
         assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"
 
     def test_isin_empty(self):
-        assert col("id").isin([]).to_sql() == "id IN ()"
+        assert col("id").isin([]).to_sql() == "false"
 
     def test_isin_filter(self, simple_table):
         result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()

python/python/tests/test_first_class_function_slice1.py

@@ -37,6 +37,21 @@ def job_result(name: str) -> dict:
     return json.loads(fixture(name))["result"]
 
 
+def assert_no_secret_values(value):
+    if isinstance(value, dict):
+        for key, child in value.items():
+            assert key not in {
+                "secret_value",
+                "secret_values",
+                "resolved_secret",
+                "resolved_secrets",
+            }
+            assert_no_secret_values(child)
+    elif isinstance(value, list):
+        for child in value:
+            assert_no_secret_values(child)
+
+
 def test_public_function_values_are_in_api_reference():
     docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
     rendered = docs.read_text()
@@ -94,6 +109,7 @@ def test_function_version_identity_is_immutable_and_exact():
     version = FunctionVersion.from_json(json.dumps(value))
     assert version.name == "embed"
     assert version.version == "fv_01K3EXACT"
+    assert version.required_secrets == ("HF_TOKEN",)
 
     with pytest.raises((TypeError, ValueError)):
         version.version = "fv_changed"
@@ -276,6 +292,15 @@ def test_refresh_result_rejects_non_u64_values(field):
         RefreshColumnResult.from_json(json.dumps(value))
 
 
+def test_canonical_client_values_contain_secret_names_only():
+    version = FunctionVersion.from_json(
+        json.dumps(job_result("remote_function_job.json"))
+    )
+    canonical = json.loads(version.to_canonical_json())
+    assert canonical["required_secrets"] == ["HF_TOKEN"]
+    assert_no_secret_values(canonical)
+
+
 class _FunctionDeclarationInner:
     def __init__(self):
         self.calls = []

python/python/tests/test_first_class_function_slice2.py

@@ -19,7 +19,13 @@ import pyarrow as pa
 import pytest
 
 import lancedb
-from lancedb.functions import UdfDefinition, udf
+from lancedb.functions import (
+    _MAX_FUNCTION_SECRET_VALUE_BYTES,
+    _MAX_FUNCTION_SECRET_VALUES_BYTES,
+    FunctionRegistrationRequest,
+    UdfDefinition,
+    udf,
+)
 
 THRESHOLD = 20
 _CACHE = None
@@ -39,12 +45,28 @@ FIXTURES = (
 @udf(
     pip=["numpy>=2"],
     env={"MODE": "test"},
+    secrets=["API_TOKEN"],
     python_version="3.12",
 )
 def normalize_score(value: float) -> float:
     return value / 100.0
 
 
+def _assert_no_secret_values(value):
+    if isinstance(value, dict):
+        for key, child in value.items():
+            assert key not in {
+                "secret_value",
+                "secret_values",
+                "resolved_secret",
+                "resolved_secrets",
+            }
+            _assert_no_secret_values(child)
+    elif isinstance(value, list):
+        for child in value:
+            _assert_no_secret_values(child)
+
+
 def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
     assert isinstance(normalize_score, UdfDefinition)
     assert normalize_score(25.0) == 0.25
@@ -59,6 +81,8 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
         "kind": "scalar_to_arrow_batch",
         "version": 1,
     }
+    assert request["required_secrets"] == ["API_TOKEN"]
+    _assert_no_secret_values(request)
 
 
 def _run_packaged(definition, *args):
@@ -372,6 +396,7 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
         output_schema=None,
         pip=(),
         env={},
+        secrets=(),
         python_version=None,
     )
     with pytest.raises(ValueError, match="binds that name to another value"):
@@ -526,13 +551,54 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
             return value
 
 
+def test_secret_names_are_canonical_and_disjoint_from_environment():
+    @udf(secrets=["Z_TOKEN", "A_TOKEN", "Z_TOKEN"])
+    def canonical_secrets(value: int) -> int:
+        return value
+
+    assert canonical_secrets.registration_request.required_secrets == (
+        "A_TOKEN",
+        "Z_TOKEN",
+    )
+
+    with pytest.raises(ValueError, match="must be disjoint"):
+
+        @udf(env={"TOKEN": "plaintext"}, secrets=["TOKEN"])
+        def overlapping(value: int) -> int:
+            return value
+
+
+def test_declared_secret_api_still_requires_explicit_create_values():
+    @udf(secrets=["API_TOKEN"])
+    def declared_secret(value: int) -> int:
+        return value
+
+    with pytest.raises(ValueError, match="missing"):
+        declared_secret._submission_json(None)
+    submission = json.loads(
+        declared_secret._submission_json({"API_TOKEN": "explicit-secret"})
+    )
+    assert submission["required_secrets"] == ["API_TOKEN"]
+    assert submission["secret_values"] == {"API_TOKEN": "explicit-secret"}
+
+
+def test_no_secrets_preserve_canonical_registration_shape():
+    @udf
+    def no_secrets(value: int) -> int:
+        return value
+
+    canonical = json.loads(no_secrets.registration_request.to_canonical_json())
+    assert "required_secrets" not in canonical
+    assert json.loads(no_secrets._submission_json(None)) == canonical
+
+
 def test_local_function_catalog_operations_are_not_supported(tmp_path):
     db = lancedb.connect(tmp_path)
     message = "Function catalog operations are not supported by this database"
     with pytest.raises(NotImplementedError, match=message):
-        db.create_function(normalize_score)
+        db.create_function(normalize_score, secrets={"API_TOKEN": "value"})
     with pytest.raises(NotImplementedError, match=message):
-        db.create_function_async(normalize_score)
+        db.create_function_async(normalize_score, secrets={"API_TOKEN": "value"})
     with pytest.raises(NotImplementedError, match=message):
         db.get_function("normalize_score", version="fv_exact")
 
@@ -562,6 +628,7 @@ def _mock_remote_function_catalog():
                     "runtime": body["runtime"],
                     "runtime_digest": "sha256:runtime",
                     "environment_digest": "sha256:environment",
+                    "required_secrets": body.get("required_secrets", []),
                     "created_at": "2026-08-21T00:00:00Z",
                 }
                 response = {"job_id": "job-register"}
@@ -608,7 +675,9 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
             host_override=host,
             client_config={"retry_config": {"retries": 0}},
         )
-        registration = db.create_function_async(normalize_score)
+        registration = db.create_function_async(
+            normalize_score, secrets={"API_TOKEN": "secret-value"}
+        )
         assert registration.id == "job-register"
         created = registration.wait()
         reopened = db.get_function("normalize_score", version=created.version)
@@ -617,9 +686,18 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
     assert reopened.name == "normalize_score"
     assert reopened.version == "fv_exact"
     create_request = state["requests"][0][1]
-    assert create_request == json.loads(
+    expected = json.loads(normalize_score.registration_request.to_canonical_json())
+    expected["secret_values"] = {"API_TOKEN": "secret-value"}
+    assert create_request == expected
+    durable_request = FunctionRegistrationRequest.from_json(json.dumps(create_request))
+    assert not hasattr(durable_request, "secret_values")
+    assert "secret_values" not in json.loads(durable_request.to_canonical_json())
+    assert "secret_values" not in json.loads(
         normalize_score.registration_request.to_canonical_json()
     )
+    assert "secret-value" not in repr(normalize_score)
+    assert "secret-value" not in repr(normalize_score.registration_request)
+    assert not hasattr(created, "secret_values")
 
 
 def test_blocking_remote_registration_returns_function_version():
@@ -630,7 +708,9 @@ def test_blocking_remote_registration_returns_function_version():
             host_override=host,
             client_config={"retry_config": {"retries": 0}},
         )
-        created = db.create_function(normalize_score)
+        created = db.create_function(
+            normalize_score, secrets={"API_TOKEN": "blocking-secret"}
+        )
 
     assert created.name == "normalize_score"
     assert created.version == "fv_exact"
@@ -638,3 +718,121 @@ def test_blocking_remote_registration_returns_function_version():
         "/v1/functions/create",
         "/v1/jobs/describe",
     ]
+    assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "blocking-secret"}
+
+
+@pytest.mark.parametrize(
+    ("secret_values", "error_type", "message"),
+    [
+        (None, ValueError, "missing"),
+        ({}, ValueError, "missing"),
+        ({"OTHER": "value"}, ValueError, "missing.*unexpected"),
+        ({"API_TOKEN": ""}, ValueError, "non-empty"),
+        ({"API_TOKEN": "bad\0value"}, ValueError, "NUL"),
+        ({"API_TOKEN": 123}, TypeError, "must be a string"),
+        ([("API_TOKEN", "value")], TypeError, "must be a mapping"),
+    ],
+)
+def test_secret_values_are_validated_before_remote_request(
+    secret_values, error_type, message
+):
+    with _mock_remote_function_catalog() as (host, state):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        with pytest.raises(error_type, match=message):
+            db.create_function_async(normalize_score, secrets=secret_values)
+        assert state["requests"] == []
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "x" * _MAX_FUNCTION_SECRET_VALUE_BYTES,
+        "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8"))),
+    ],
+    ids=["ascii", "multibyte"],
+)
+def test_secret_value_accepts_exact_utf8_byte_limit(value):
+    submission = json.loads(normalize_score._submission_json({"API_TOKEN": value}))
+    assert submission["secret_values"]["API_TOKEN"] == value
+    assert len(value.encode("utf-8")) == _MAX_FUNCTION_SECRET_VALUE_BYTES
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "x" * (_MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
+        "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")) + 1),
+    ],
+    ids=["ascii", "multibyte"],
+)
+def test_secret_value_rejects_over_utf8_byte_limit_before_json_construction(
+    monkeypatch, value
+):
+    def fail_if_json_construction_starts(self):
+        pytest.fail("oversized secret reached JSON construction")
+
+    monkeypatch.setattr(
+        FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
+    )
+    with pytest.raises(ValueError, match=r"exceeds the 65536-byte limit"):
+        normalize_score._submission_json({"API_TOKEN": value})
+
+
+def test_secret_values_accept_exact_aggregate_utf8_byte_limit(monkeypatch):
+    names = tuple(f"SECRET_{index}" for index in range(8))
+    value = "é" * (_MAX_FUNCTION_SECRET_VALUE_BYTES // len("é".encode("utf-8")))
+    values = {name: value for name in names}
+    monkeypatch.setattr(
+        normalize_score,
+        "_request",
+        normalize_score._request._copy(update={"required_secrets": names}),
+    )
+
+    submission = json.loads(normalize_score._submission_json(values))
+
+    assert submission["secret_values"] == values
+    assert sum(len(item.encode("utf-8")) for item in values.values()) == (
+        _MAX_FUNCTION_SECRET_VALUES_BYTES
+    )
+
+
+def test_secret_values_reject_aggregate_over_limit_before_construction(monkeypatch):
+    names = tuple(f"SECRET_{index}" for index in range(9))
+    values = {name: "x" * _MAX_FUNCTION_SECRET_VALUE_BYTES for name in names}
+    monkeypatch.setattr(
+        normalize_score,
+        "_request",
+        normalize_score._request._copy(update={"required_secrets": names}),
+    )
+
+    def fail_if_json_construction_starts(self):
+        pytest.fail("oversized aggregate reached JSON construction")
+
+    monkeypatch.setattr(
+        FunctionRegistrationRequest, "_known_dict", fail_if_json_construction_starts
+    )
+    with pytest.raises(ValueError, match=r"exceed.*524288-byte request limit"):
+        normalize_score._submission_json(values)
+
+
+@pytest.mark.asyncio
+async def test_async_remote_registration_submits_secret_values_only_once():
+    with _mock_remote_function_catalog() as (host, state):
+        db = await lancedb.connect_async(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            client_config={"retry_config": {"retries": 0}},
+        )
+        registration = await db.create_function_async(
+            normalize_score, secrets={"API_TOKEN": "async-secret"}
+        )
+        created = await registration.wait()
+
+    assert state["requests"][0][1]["secret_values"] == {"API_TOKEN": "async-secret"}
+    assert not hasattr(created, "secret_values")

python/python/tests/test_query.py

@@ -675,6 +675,21 @@ def test_distance_range(table: lancedb.table.Table):
     assert res["_distance"].to_pylist() == [min_dist, max_dist]
 
 
+@pytest.mark.parametrize("expression", ["1 - _distance", "1.0 - _distance"])
+def test_select_arithmetic_with_distance(table, expression):
+    result = (
+        table.search([10, 10])
+        .select({"similarity": expression, "_distance": "_distance"})
+        .distance_type("cosine")
+        .to_arrow()
+    )
+
+    assert result.schema.field("similarity").type == pa.float32()
+    assert result["similarity"].to_pylist() == pytest.approx(
+        [1 - distance for distance in result["_distance"].to_pylist()]
+    )
+
+
 @pytest.mark.asyncio
 async def test_distance_range_async(table_async: AsyncTable):
     q = [0, 0]
@@ -897,6 +912,23 @@ def test_query_builder_batches(table):
     assert rs_list["id"][1] == 2
 
 
+def test_batch_vector_query_shares_filtered_flat_scan(table):
+    query = (
+        table.search([[1.0, 2.0], [3.0, 4.0]])
+        .where("id > 0", prefilter=True)
+        .limit(1)
+        .select(["id"])
+    )
+
+    plan = query.explain_plan(verbose=True)
+    assert "KNNVectorDistance: queries=2" in plan
+    assert "UnionExec" not in plan
+
+    results = query.to_arrow()
+    assert len(results) == 2
+    assert results["query_index"].to_pylist() == [0, 1]
+
+
 def test_dynamic_projection(table):
     rs = (
         LanceVectorQueryBuilder(table, [0, 0], "vector")

python/python/tests/test_table.py

@@ -11,6 +11,7 @@ import warnings
 import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
+from decimal import Decimal
 from time import sleep
 from typing import List
 from unittest.mock import patch
@@ -336,6 +337,21 @@ async def test_update_async(mem_db_async: AsyncConnection):
     assert await table.count_rows("id == 10") == 1
 
 
+@pytest.mark.asyncio
+async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
+    values = ["5", "4.66e-84", "it's"]
+    table = await mem_db_async.create_table(
+        "update_expr_literals",
+        data=[{"field": value, "result": "original"} for value in values],
+    )
+
+    for value in values:
+        update_res = await table.update({"result": value}, where=col("field") == value)
+        assert update_res.rows_updated == 1
+
+    assert (await table.to_arrow())["result"].to_pylist() == values
+
+
 def test_create_table(mem_db: DBConnection):
     schema = pa.schema(
         {
@@ -2343,6 +2359,148 @@ def test_update(mem_db: DBConnection):
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
+def test_update_expr_filter_literals(mem_db: DBConnection):
+    values = ["5", "4.66e-84", "it's"]
+    table = mem_db.create_table(
+        "update_expr_literals",
+        data=[{"field": value, "result": "original"} for value in values],
+    )
+
+    for value in values:
+        update_res = table.update(where=col("field") == value, values={"result": value})
+        assert update_res.rows_updated == 1
+
+    assert table.to_arrow()["result"].to_pylist() == values
+
+
+def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
+    low = Decimal("1.234567890123456789")
+    high = Decimal("1.234567890123456790")
+    decimal_schema = pa.schema(
+        [("val", pa.decimal128(19, 18)), ("result", pa.string())]
+    )
+    decimal_table = mem_db.create_table(
+        "update_expr_decimal",
+        pa.table(
+            {"val": [low, high], "result": ["old", "old"]},
+            schema=decimal_schema,
+        ),
+    )
+    predicate = col("val") < lit(high)
+    assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
+    result = decimal_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    keyword_table = mem_db.create_table(
+        "update_expr_keyword", [{"null": 1, "result": "old"}]
+    )
+    predicate = col("null") == 1
+    assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
+    result = keyword_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    empty_in_table = mem_db.create_table(
+        "update_expr_empty_in", [{"id": 1, "result": "old"}]
+    )
+    predicate = col("id").isin([])
+    assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
+    result = empty_in_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 0
+
+    marker = "__lancedb_binary_placeholder_0__"
+    binary_schema = pa.schema(
+        [("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
+    )
+    binary_table = mem_db.create_table(
+        "update_expr_binary",
+        pa.table(
+            {
+                "payload": [b"\x01", b"\x02"],
+                "text": ["other", marker],
+                "result": ["old", "old"],
+            },
+            schema=binary_schema,
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
+    assert binary_table.search().where(predicate).to_arrow().num_rows == 2
+    result = binary_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 2
+
+    nonfinite_table = mem_db.create_table(
+        "update_expr_nonfinite",
+        [{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
+    )
+    predicate = col("x") < float("inf")
+    assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
+    result = nonfinite_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 2
+
+    float16_table = mem_db.create_table(
+        "update_expr_float16",
+        [{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
+    )
+    predicate = col("x").cast(pa.float16()) < 2.0
+    assert float16_table.search().where(predicate).to_arrow().num_rows == 1
+    result = float16_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    string_cast_table = mem_db.create_table(
+        "update_expr_string_cast",
+        [{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
+    )
+    predicate = col("x").cast("string") == "1"
+    assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
+    result = string_cast_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    quoted_identifier_schema = pa.schema(
+        [("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
+    )
+    quoted_identifier_table = mem_db.create_table(
+        "update_expr_quoted_identifier",
+        pa.table(
+            {"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
+            schema=quoted_identifier_schema,
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
+    assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
+    result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    decimal256_schema = pa.schema(
+        [("val", pa.decimal256(40, 2)), ("result", pa.string())]
+    )
+    decimal256_table = mem_db.create_table(
+        "update_expr_decimal256",
+        pa.table(
+            {
+                "val": [Decimal("1.00"), Decimal("3.00")],
+                "result": ["old", "old"],
+            },
+            schema=decimal256_schema,
+        ),
+    )
+    predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
+    assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
+    result = decimal256_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 1
+
+    binary_empty_table = mem_db.create_table(
+        "update_expr_binary_empty",
+        pa.table(
+            {"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
+            schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
+        ),
+    )
+    predicate = (col("payload") == lit(b"\x01")).isin([])
+    assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
+    assert predicate.to_sql() == "false"
+    result = binary_empty_table.update(where=predicate, values={"result": "new"})
+    assert result.rows_updated == 0
+
+
 def test_update_with_arrow_scalar(mem_db: DBConnection):
     schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
     table = mem_db.create_table("my_table", schema=schema)

python/python/tests/test_util.py

@@ -7,6 +7,7 @@ import pathlib
 from typing import Optional
 
 import lance
+from lance.blob import BlobType as LanceBlobType
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.embeddings.base import EmbeddingFunctionConfig
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
@@ -907,6 +908,165 @@ def test_cast_to_target_schema():
     assert output == expected
 
 
+def test_cast_to_target_schema_coerces_binary_to_blob_v2():
+    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
+    target = pa.schema([lancedb.blob("image")])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert type(image.type) is lancedb.BlobType
+    assert image.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
+    storage = lancedb.blob("image").type.storage_type
+    target = pa.schema(
+        [
+            pa.field(
+                "image",
+                storage,
+                metadata={
+                    b"ARROW:extension:name": b"lance.blob.v2",
+                    b"ARROW:extension:metadata": b"",
+                },
+            )
+        ]
+    )
+    data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert not isinstance(image.type, pa.ExtensionType)
+    assert image.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_nested_binary_blob():
+    data = pa.table(
+        {
+            "info": pa.array(
+                [{"blob": b"hello"}, {"blob": None}],
+                type=pa.struct([pa.field("blob", pa.binary())]),
+            )
+        }
+    )
+    target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    blob = output["info"].chunk(0).field("blob")
+    assert type(blob.type) is lancedb.BlobType
+    assert blob.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
+    data = pa.table(
+        {"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
+    )
+    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    images = output["images"].chunk(0)
+    assert images.type.value_field.name == "image"
+    assert type(images.type.value_type) is lancedb.BlobType
+    assert images.to_pylist()[1] is None
+    assert images.values.storage.to_pylist() == [
+        {"data": b"a", "uri": None, "position": None, "size": None},
+        {"data": b"b", "uri": None, "position": None, "size": None},
+    ]
+
+
+def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
+    child = pa.field("image", pa.binary())
+    source = pa.ListArray.from_arrays(
+        pa.array([0, 2, 4], type=pa.int32()),
+        pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
+        mask=pa.array([False, True]),
+    ).cast(pa.list_(child))
+    target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
+
+    output = _cast_to_target_schema(
+        pa.table({"images": source}).to_reader(), target
+    ).read_all()
+
+    images = output["images"].chunk(0)
+    assert images.to_pylist()[1] is None
+    assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
+
+
+def test_fixed_size_list_blob_coercion_keeps_null_rows():
+    child = pa.field("frame", pa.binary())
+    source = (
+        pa.FixedSizeListArray.from_arrays(
+            pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
+        )
+        .take(pa.array([0, None], type=pa.int32()))
+        .cast(pa.list_(child, 2))
+    )
+    target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
+
+    output = _cast_to_target_schema(
+        pa.table({"frames": source}).to_reader(), target
+    ).read_all()
+
+    frames = output["frames"].chunk(0)
+    assert frames.to_pylist()[1] is None
+    assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
+
+
+def test_cast_to_target_schema_accepts_pylance_blob_v2():
+    target_type = lancedb.BlobType()
+    source = lance.blob_array([b"hello", None])
+    assert type(source.type) is LanceBlobType
+    assert type(source.type) is type(target_type)
+    data = pa.table({"image": source})
+    target = pa.schema([pa.field("image", target_type)])
+
+    output = _cast_to_target_schema(data.to_reader(), target).read_all()
+
+    image = output["image"].chunk(0)
+    assert type(image.type) is LanceBlobType
+    assert image.type == target_type
+    assert image.storage.to_pylist() == [
+        {"data": b"hello", "uri": None, "position": None, "size": None},
+        None,
+    ]
+
+
+def test_cast_to_target_schema_rejects_different_blob_v2_class():
+    class OtherBlobType(pa.ExtensionType):
+        def __init__(self):
+            super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
+
+        def __arrow_ext_serialize__(self) -> bytes:
+            return b""
+
+        @classmethod
+        def __arrow_ext_deserialize__(
+            cls, storage_type: pa.DataType, serialized: bytes
+        ) -> "OtherBlobType":
+            return cls()
+
+    storage = lance.blob_array([b"hello"]).storage
+    source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
+    data = pa.table({"image": source})
+    target = pa.schema([lancedb.blob("image")])
+
+    with pytest.raises(pa.ArrowTypeError, match="different extension type"):
+        _cast_to_target_schema(data.to_reader(), target).read_all()
+
+
 def test_sanitize_data_stream():
     # Make sure we don't collect the whole stream when running sanitize_data
     schema = pa.schema({"a": pa.int32()})

python/src/expr.rs

@@ -130,6 +130,14 @@ impl PyExpr {
 
     // ── utilities ────────────────────────────────────────────────────────────
 
+    /// Return the referenced column name for a bare column expression.
+    fn column_name(&self) -> Option<String> {
+        match &self.0 {
+            DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()),
+            _ => None,
+        }
+    }
+
     /// Render the expression as a SQL string (useful for debugging).
     fn to_sql(&self) -> PyResult<String> {
         lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string()))

python/src/query.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -325,6 +326,7 @@ pub struct PyQueryRequest {
     pub filter: Option<PyQueryFilter>,
     pub full_text_search: Option<PyLanceDB<FtsQuery>>,
     pub select: PySelect,
+    pub select_source_columns: Option<HashMap<String, String>>,
     pub fast_search: Option<bool>,
     pub with_row_id: Option<bool>,
     pub use_lsm: Option<bool>,
@@ -355,6 +357,7 @@ impl From<AnyQuery> for PyQueryRequest {
                 full_text_search: query_request
                     .full_text_search
                     .map(|fts| PyLanceDB(fts.query)),
+                select_source_columns: PySelect::source_columns(&query_request.select),
                 select: PySelect(query_request.select),
                 fast_search: Some(query_request.fast_search),
                 with_row_id: Some(query_request.with_row_id),
@@ -380,6 +383,7 @@ impl From<AnyQuery> for PyQueryRequest {
                 offset: vector_query.base.offset,
                 filter: vector_query.base.filter.map(PyQueryFilter),
                 full_text_search: None,
+                select_source_columns: PySelect::source_columns(&vector_query.base.select),
                 select: PySelect(vector_query.base.select),
                 fast_search: Some(vector_query.base.fast_search),
                 with_row_id: Some(vector_query.base.with_row_id),
@@ -412,6 +416,25 @@ impl From<AnyQuery> for PyQueryRequest {
 #[derive(Clone)]
 pub struct PySelect(Select);
 
+impl PySelect {
+    fn source_columns(select: &Select) -> Option<HashMap<String, String>> {
+        match select {
+            Select::Expr(pairs) => Some(
+                pairs
+                    .iter()
+                    .filter_map(|(output, expr)| match expr {
+                        lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => {
+                            Some((output.clone(), column.name.clone()))
+                        }
+                        _ => None,
+                    })
+                    .collect(),
+            ),
+            _ => None,
+        }
+    }
+}
+
 impl<'py> IntoPyObject<'py> for PySelect {
     type Target = PyAny;
     type Output = Bound<'py, Self::Target>;

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.10"
+version = "0.38.0-beta.11"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/lancedb/src/database/listing.rs

@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -281,6 +281,22 @@ impl std::fmt::Display for ListingDatabase {
 }
 
 const LANCE_EXTENSION: &str = "lance";
+
+/// The table a listed child of the database names, or `None` if the child is not a table.
+///
+/// A table is the directory `<name>.lance`; a loose file or any other directory under the
+/// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
+/// caller rather than per child.
+/// The table a listed child directory holds, or `None` if it is not a table at all.
+///
+/// Only directories are considered, so a loose object named like a table is not one.
+fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
+    location
+        .filename()?
+        .strip_suffix(dir_suffix)
+        .map(String::from)
+        .filter(|name| !name.is_empty())
+}
 const ENGINE: &str = "engine";
 const MIRRORED_STORE: &str = "mirroredStore";
 
@@ -944,51 +960,72 @@ impl Database for ListingDatabase {
         Ok(f)
     }
 
+    /// List the tables in the database, a page at a time.
+    ///
+    /// The page_token is opaque, unlike the `start_after` parameter of [`Self::table_names()`].
+    ///
+    /// When there are no more results, the returned page_token will be None.
+    ///
+    /// `limit` is the maximum number of tables to return in the response. But it is possible
+    /// for the response to contain fewer than `limit` tables, even when there are more tables
+    /// to return. Clients should check the returned page_token to determine if there are
+    /// more results, rather than relying on the number of tables returned.
+    ///
+    /// The order that results are returned in not guaranteed to be stable across calls,
+    /// so clients should not rely on it.
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
         if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
             return self.namespace_database().list_tables(request).await;
         }
-        let mut f = self
-            .object_store
-            .read_dir(self.base_path.clone())
-            .await?
-            .iter()
-            .map(Path::new)
-            .filter(|path| {
-                let is_lance = path
-                    .extension()
-                    .and_then(|e| e.to_str())
-                    .map(|e| e == LANCE_EXTENSION);
-                is_lance.unwrap_or(false)
-            })
-            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect::<Vec<String>>();
-        f.sort();
+        let limit = request.limit.map(|limit| limit.max(0) as usize);
+        let dir_suffix = format!(".{LANCE_EXTENSION}");
+        let mut tables = Vec::new();
+        let mut page_token = request.page_token.filter(|token| !token.is_empty());
 
-        // Handle pagination with page_token
-        if let Some(ref page_token) = request.page_token {
-            let index = f
-                .iter()
-                .position(|name| name.as_str() > page_token.as_str())
-                .unwrap_or(f.len());
-            f.drain(0..index);
+        // A page of nothing: the store rejects a limit of zero, and no table was handed over
+        // for a token to resume after.
+        if limit == Some(0) {
+            return Ok(ListTablesResponse {
+                context: None,
+                tables,
+                page_token: None,
+            });
         }
 
-        // Determine if there's a next page. The token is the last name of this page,
-        // not the first of the next one: the next page resumes strictly after the
-        // token, so naming the next page's first entry would skip it.
-        let next_page_token = match request.limit {
-            Some(limit) if f.len() > limit as usize => {
-                f.truncate(limit as usize);
-                f.last().cloned()
+        loop {
+            // Ask only for what the page still has room for, so a database holding more
+            // than one page costs one request per page rather than one per table.
+            let listing = self
+                .object_store
+                .read_dir_page(
+                    self.base_path.clone(),
+                    ReadDirOptions {
+                        page_token: page_token.take(),
+                        limit: limit.map(|limit| limit - tables.len()),
+                    },
+                )
+                .await?;
+            page_token = listing.page_token;
+            // Only child directories can be tables, and the store already separates them
+            // out, so the objects in the page are not looked at.
+            tables.extend(
+                listing
+                    .result
+                    .common_prefixes
+                    .iter()
+                    .filter_map(|location| table_name(location, &dir_suffix)),
+            );
+            // Children that are not tables leave the page short of the limit, so keep
+            // going until the page is full or the database runs out.
+            if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
+                break;
             }
-            _ => None,
-        };
+        }
 
         Ok(ListTablesResponse {
             context: None,
-            tables: f,
-            page_token: next_page_token,
+            tables,
+            page_token,
         })
     }
 
@@ -1484,6 +1521,182 @@ mod tests {
     use tokio::sync::Barrier;
     use tokio::time::timeout;
 
+    async fn create_tables(db: &ListingDatabase, names: &[&str]) {
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        for name in names {
+            db.create_table(CreateTableRequest {
+                name: name.to_string(),
+                namespace_path: vec![],
+                data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
+                mode: CreateTableMode::Create,
+                write_options: Default::default(),
+                location: None,
+                namespace_client: None,
+            })
+            .await
+            .unwrap();
+        }
+    }
+
+    /// Every table in the database, taken `limit` at a time, which is how a caller walks a
+    /// listing: the token ends the walk, never a short page.
+    async fn walk(db: &ListingDatabase, limit: Option<i32>) -> Vec<String> {
+        let mut seen = Vec::new();
+        let mut page_token = None;
+        loop {
+            let page = db
+                .list_tables(ListTablesRequest {
+                    limit,
+                    page_token,
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            seen.extend(page.tables);
+            page_token = page.page_token;
+            if page_token.is_none() {
+                return seen;
+            }
+            assert!(
+                seen.len() < 100,
+                "the walk is serving tables more than once"
+            );
+        }
+    }
+
+    /// Paging with the returned token has to visit every table exactly once, whatever the
+    /// page size, with nothing lost or repeated at a boundary.
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn test_list_tables_pages_over_every_table_once(#[values(1, 2, 3, 5, 10)] limit: i32) {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b", "c", "d", "e"]).await;
+
+        assert_eq!(walk(&db, Some(limit)).await, vec!["a", "b", "c", "d", "e"]);
+    }
+
+    /// The token is opaque: it is whatever resumes the store the database sits on, not a
+    /// table name. Callers hand it back and nothing else.
+    ///
+    /// Nothing validates a token, so one invented by a caller is read as a position rather
+    /// than refused — which is why the token has to come back from a previous page.
+    #[tokio::test]
+    async fn test_the_page_token_is_not_a_table_name() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b", "c"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(1),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a"]);
+        let token = page.page_token.expect("two tables are still to come");
+        assert_ne!(token, "a");
+
+        // Handing it back is the only thing a caller does with it, and it resumes.
+        let rest = db
+            .list_tables(ListTablesRequest {
+                page_token: Some(token),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(rest.tables, vec!["b", "c"]);
+    }
+
+    /// A limit the listing does not fill leaves no token behind, so a caller paging by token
+    /// stops without asking for an empty page.
+    #[tokio::test]
+    async fn test_a_listing_that_runs_out_has_no_token() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(10),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a", "b"]);
+        assert_eq!(page.page_token, None);
+    }
+
+    /// An empty page token means "from the start", which is how a client looping on a token
+    /// spells its first request.
+    #[tokio::test]
+    async fn test_an_empty_page_token_lists_from_the_start() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["a", "b"]).await;
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                page_token: Some(String::new()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["a", "b"]);
+    }
+
+    /// Listing follows the order the object store lists directories in, so a name that
+    /// extends another comes first: the `-` of `users-archive.lance` sorts below the `.` of
+    /// `users.lance`. Pagination pushes its cursor into the list request, so it cannot report
+    /// an order other than the one it resumes in.
+    #[tokio::test]
+    async fn test_listing_order_follows_the_store_not_the_table_name() {
+        let (_tempdir, db) = setup_database().await;
+        create_tables(&db, &["users", "users-archive", "users.old"]).await;
+
+        assert_eq!(
+            walk(&db, None).await,
+            vec!["users-archive", "users", "users.old"]
+        );
+        // And paging reports the same order, so a walk sees each table once.
+        assert_eq!(
+            walk(&db, Some(1)).await,
+            vec!["users-archive", "users", "users.old"]
+        );
+    }
+
+    /// Only directories named `<name>.lance` are tables; loose files and other directories
+    /// under the database prefix are not. A page spent on them is filled from the next one,
+    /// so a page holding only non-tables does not read as an empty database.
+    #[tokio::test]
+    async fn test_listing_ignores_non_table_children() {
+        let (tempdir, db) = setup_database().await;
+        create_tables(&db, &["real"]).await;
+        std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
+        create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
+
+        let page = db
+            .list_tables(ListTablesRequest {
+                limit: Some(1),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(page.tables, vec!["real"]);
+    }
+
+    #[tokio::test]
+    async fn listing_ignores_empty_table_name() {
+        let (tempdir, db) = setup_database().await;
+        create_dir_all(tempdir.path().join(".lance")).unwrap();
+        let page = db.list_tables(ListTablesRequest::default()).await.unwrap();
+        assert!(
+            page.tables.is_empty(),
+            "invalid empty table name was listed"
+        );
+    }
+
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
         let tempdir = tempdir().unwrap();
         let uri = tempdir.path().to_str().unwrap();

rust/lancedb/src/expr.rs

@@ -19,6 +19,7 @@
 
 mod sql;
 
+pub(crate) use sql::canonicalize_sql_predicate;
 pub use sql::expr_to_sql_string;
 
 use std::sync::Arc;
@@ -156,7 +157,7 @@ mod tests {
         use datafusion_common::ScalarValue;
         let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "(data = X'CAFE')");
+        assert_eq!(sql, "(`data` = X'CAFE')");
     }
 
     #[test]
@@ -166,7 +167,7 @@ mod tests {
         let int_expr = col("id").gt(lit(5i64));
         let combined = bin_expr.and(int_expr);
         let sql = expr_to_sql_string(&combined).unwrap();
-        assert_eq!(sql, "((data = X'01') AND (id > 5))");
+        assert_eq!(sql, "((`data` = X'01') AND (id > 5))");
     }
 
     #[test]
@@ -184,7 +185,7 @@ mod tests {
         // serialized correctly (regression test for placeholder rewrite path).
         let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "contains(data, X'FF')");
+        assert_eq!(sql, "contains(`data`, X'FF')");
     }
 
     #[test]
@@ -195,7 +196,7 @@ mod tests {
             .eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd]))))
             .not();
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "NOT (data = X'ABCD')");
+        assert_eq!(sql, "NOT (`data` = X'ABCD')");
     }
 
     #[test]
@@ -205,6 +206,122 @@ mod tests {
         assert!(sql.contains("IN"), "expected IN in: {}", sql);
     }
 
+    #[test]
+    fn test_empty_is_in() {
+        let expr = is_in(col("id"), vec![]);
+        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
+    }
+
+    #[test]
+    fn test_empty_is_in_discards_binary_children() {
+        use datafusion_common::ScalarValue;
+
+        let expr = is_in(
+            col("payload").eq(lit(ScalarValue::Binary(Some(vec![0x01])))),
+            vec![],
+        );
+        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
+    }
+
+    #[test]
+    fn test_keyword_identifier() {
+        let expr = col("null").eq(lit(1i64));
+        assert_eq!(expr_to_sql_string(&expr).unwrap(), "(`null` = 1)");
+    }
+
+    #[test]
+    fn test_decimal_literal_preserves_type() {
+        use datafusion_common::ScalarValue;
+
+        let expr = col("val").lt(lit(ScalarValue::Decimal128(
+            Some(1_234_567_890_123_456_790),
+            19,
+            18,
+        )));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert_eq!(
+            sql,
+            "(val < arrow_cast('1.234567890123456790', 'Decimal128(19, 18)'))"
+        );
+    }
+
+    #[test]
+    fn test_non_finite_float_literal_preserves_type() {
+        let expr = col("x").lt(lit(f64::INFINITY));
+        assert_eq!(
+            expr_to_sql_string(&expr).unwrap(),
+            "(x < arrow_cast('inf', 'Float64'))"
+        );
+    }
+
+    #[test]
+    fn test_cast_uses_arrow_type_name() {
+        let string = expr_cast(col("x"), DataType::Utf8);
+        assert_eq!(
+            expr_to_sql_string(&string).unwrap(),
+            "arrow_cast(x, 'Utf8')"
+        );
+
+        let int32 = expr_cast(col("x"), DataType::Int32);
+        assert_eq!(
+            expr_to_sql_string(&int32).unwrap(),
+            "arrow_cast(x, 'Int32')"
+        );
+
+        let expr = expr_cast(col("x"), DataType::Float16).lt(lit(2.0));
+        assert_eq!(
+            expr_to_sql_string(&expr).unwrap(),
+            "(arrow_cast(x, 'Float16') < 2.0)"
+        );
+
+        let decimal = expr_cast(lit("2.00"), DataType::Decimal256(40, 2));
+        assert_eq!(
+            expr_to_sql_string(&decimal).unwrap(),
+            "arrow_cast('2.00', 'Decimal256(40, 2)')"
+        );
+    }
+
+    #[test]
+    fn test_binary_placeholder_does_not_rewrite_user_string() {
+        use datafusion_common::ScalarValue;
+
+        let marker = "__lancedb_binary_placeholder_0__";
+        let expr = col("payload")
+            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
+            .or(col("text").eq(lit(marker)));
+        assert_eq!(
+            expr_to_sql_string(&expr).unwrap(),
+            "((payload = X'01') OR (`text` = '__lancedb_binary_placeholder_0__'))"
+        );
+    }
+
+    #[test]
+    fn test_binary_binding_skips_quoted_identifiers() {
+        use datafusion_common::ScalarValue;
+
+        let expr = col("payload")
+            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
+            .and(col("odd'name").eq(lit(1i64)))
+            .and(col("odd`'name").eq(lit(2i64)));
+        assert_eq!(
+            expr_to_sql_string(&expr).unwrap(),
+            "(((payload = X'01') AND (`odd'name` = 1)) AND (`odd``'name` = 2))"
+        );
+    }
+
+    #[test]
+    fn test_binary_placeholder_collision_search_is_linear() {
+        use datafusion_common::ScalarValue;
+
+        let collision_shaped = format!("__lancedb_binary_placeholder_0__{}", "_".repeat(64_000));
+        let expr = col("payload")
+            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
+            .and(col("text").eq(lit(collision_shaped.clone())));
+        let sql = expr_to_sql_string(&expr).unwrap();
+        assert!(sql.contains("X'01'"));
+        assert!(sql.contains(&format!("'{collision_shaped}'")));
+    }
+
     #[test]
     fn test_multiple_binary_literals() {
         use datafusion_common::ScalarValue;

rust/lancedb/src/expr/sql.rs

@@ -1,10 +1,27 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+use std::{
+    any::TypeId,
+    collections::{HashMap, HashSet},
+};
+
+use arrow_array::types::{
+    Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
+};
+use arrow_schema::DataType;
 use datafusion_common::ScalarValue;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
 use datafusion_expr::Expr;
-use datafusion_sql::unparser::{self, dialect::Dialect};
+use datafusion_functions::core::expr_fn::{
+    arrow_cast as datafusion_arrow_cast, arrow_try_cast as datafusion_arrow_try_cast,
+};
+use datafusion_sql::sqlparser::{
+    dialect::{Dialect as SqlParserDialect, GenericDialect},
+    keywords::ALL_KEYWORDS,
+    tokenizer::{Token, Tokenizer},
+};
+use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
 
 /// Unparser dialect that matches the quoting style expected by the Lance SQL
 /// parser.  Lance uses backtick (`` ` ``) as the only delimited-identifier
@@ -19,17 +36,74 @@ use datafusion_sql::unparser::{self, dialect::Dialect};
 ///   lower-case by the SQL parser, which would break case-sensitive schemas).
 struct LanceSqlDialect;
 
-impl Dialect for LanceSqlDialect {
+impl UnparserDialect for LanceSqlDialect {
     fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
-        let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
-            || !identifier
-                .chars()
-                .enumerate()
-                .all(|(i, c)| c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()));
+        let identifier_upper = identifier.to_ascii_uppercase();
+        let needs_quote =
+            (identifier_upper != "ID" && ALL_KEYWORDS.contains(&identifier_upper.as_str()))
+                || identifier.chars().any(|c| c.is_ascii_uppercase())
+                || !identifier.chars().enumerate().all(|(i, c)| {
+                    c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
+                });
         if needs_quote { Some('`') } else { None }
     }
 }
 
+/// Lance's tokenizer dialect with SQL-standard double-quoted identifiers added.
+///
+/// Keep this deliberately small: Lance's parser wraps `GenericDialect` and
+/// delegates only identifier recognition, leaving every other dialect option at
+/// its default. In particular, `/*! ... */` remains an ordinary block comment.
+#[derive(Debug, Default)]
+struct PredicateDialect(GenericDialect);
+
+impl SqlParserDialect for PredicateDialect {
+    fn dialect(&self) -> TypeId {
+        self.0.dialect()
+    }
+
+    fn is_identifier_start(&self, ch: char) -> bool {
+        self.0.is_identifier_start(ch)
+    }
+
+    fn is_identifier_part(&self, ch: char) -> bool {
+        self.0.is_identifier_part(ch)
+    }
+
+    fn is_delimited_identifier_start(&self, ch: char) -> bool {
+        ch == '"' || ch == '`'
+    }
+}
+
+/// Canonicalize a raw SQL predicate for Lance's parser.
+///
+/// Lance wraps [`GenericDialect`] for identifier recognition while retaining the
+/// default dialect behavior for every other lexical option. [`PredicateDialect`]
+/// mirrors that contract and additionally recognizes `"` as an identifier
+/// delimiter, allowing this function to rewrite only those identifier tokens.
+pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result<String> {
+    let dialect = PredicateDialect::default();
+    let tokens = Tokenizer::new(&dialect, predicate)
+        .with_unescape(false)
+        .tokenize()
+        .map_err(|err| crate::Error::InvalidInput {
+            message: format!("invalid SQL predicate: {err}"),
+        })?;
+
+    Ok(tokens
+        .into_iter()
+        .map(|token| match token {
+            Token::Word(word) if word.quote_style == Some('"') => {
+                // with_unescape(false) retains doubled double quotes. Decode
+                // those before escaping any backticks for Lance's delimiter.
+                let identifier = word.value.replace("\"\"", "\"").replace('`', "``");
+                format!("`{identifier}`")
+            }
+            other => other.to_string(),
+        })
+        .collect())
+}
+
 /// Prefix for placeholder strings inserted in place of binary literals.  Chosen
 /// to be extremely unlikely to occur in user data.
 const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
@@ -39,24 +113,128 @@ fn bytes_to_hex_sql(bytes: &[u8]) -> String {
     format!("X'{hex}'")
 }
 
-/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal
-/// anywhere in its subtree.  DataFusion's SQL unparser cannot serialize those
-/// variants, so we route such expressions through a placeholder-substitution
-/// path that emits SQL `X'...'` byte-string literals.
-fn has_binary_literal(expr: &Expr) -> bool {
-    let mut found = false;
+fn string_literals(expr: &Expr) -> HashSet<String> {
+    let mut literals = HashSet::new();
     let _ = expr.apply(&mut |e: &Expr| {
-        if matches!(
-            e,
-            Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _)
-        ) {
-            found = true;
-            Ok(TreeNodeRecursion::Stop)
-        } else {
-            Ok(TreeNodeRecursion::Continue)
+        if let Expr::Literal(
+            ScalarValue::Utf8(Some(value))
+            | ScalarValue::LargeUtf8(Some(value))
+            | ScalarValue::Utf8View(Some(value)),
+            _,
+        ) = e
+        {
+            literals.insert(value.clone());
         }
+        Ok(TreeNodeRecursion::Continue)
     });
-    found
+    literals
+}
+
+fn typed_string_literal(value: String, data_type: DataType) -> Expr {
+    datafusion_arrow_cast(
+        Expr::Literal(ScalarValue::Utf8(Some(value)), None),
+        Expr::Literal(ScalarValue::Utf8(Some(data_type.to_string())), None),
+    )
+}
+
+fn next_binary_placeholder(user_strings: &HashSet<String>, next_id: &mut usize) -> String {
+    loop {
+        let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id);
+        *next_id += 1;
+        if !user_strings.contains(&placeholder) {
+            return placeholder;
+        }
+    }
+}
+
+fn bind_binary_literals(
+    sql: &str,
+    mut bindings: HashMap<String, Vec<u8>>,
+) -> crate::Result<String> {
+    let bytes = sql.as_bytes();
+    let mut output = Vec::with_capacity(bytes.len());
+    let mut index = 0;
+
+    // Walk SQL string tokens once. Placeholders are plain, unescaped string
+    // literals, so this remains linear even when user strings are large or
+    // deliberately resemble the placeholder prefix.
+    while index < bytes.len() {
+        if bytes[index] == b'`' {
+            let identifier_start = index;
+            index += 1;
+            let mut identifier_end = None;
+            while index < bytes.len() {
+                if bytes[index] == b'`' {
+                    if index + 1 < bytes.len() && bytes[index + 1] == b'`' {
+                        index += 2;
+                    } else {
+                        index += 1;
+                        identifier_end = Some(index);
+                        break;
+                    }
+                } else {
+                    index += 1;
+                }
+            }
+
+            let Some(identifier_end) = identifier_end else {
+                return Err(crate::Error::InvalidInput {
+                    message: "unterminated identifier while binding binary literal".to_string(),
+                });
+            };
+            output.extend_from_slice(&bytes[identifier_start..identifier_end]);
+            continue;
+        }
+
+        if bytes[index] != b'\'' {
+            output.push(bytes[index]);
+            index += 1;
+            continue;
+        }
+
+        let literal_start = index;
+        index += 1;
+        let content_start = index;
+        let mut escaped = false;
+        let mut content_end = None;
+        while index < bytes.len() {
+            if bytes[index] == b'\'' {
+                if index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
+                    escaped = true;
+                    index += 2;
+                } else {
+                    content_end = Some(index);
+                    index += 1;
+                    break;
+                }
+            } else {
+                index += 1;
+            }
+        }
+
+        let Some(content_end) = content_end else {
+            return Err(crate::Error::InvalidInput {
+                message: "unterminated string while binding binary literal".to_string(),
+            });
+        };
+
+        let placeholder = &sql[content_start..content_end];
+        if !escaped && let Some(value) = bindings.remove(placeholder) {
+            output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes());
+        } else {
+            output.extend_from_slice(&bytes[literal_start..index]);
+        }
+    }
+
+    if !bindings.is_empty() {
+        return Err(crate::Error::InvalidInput {
+            message: "failed to bind binary literal while serializing expression".to_string(),
+        });
+    }
+
+    String::from_utf8(output).map_err(|e| crate::Error::InvalidInput {
+        message: format!("failed to bind binary literal: {e}"),
+    })
 }
 
 fn run_unparser(expr: &Expr) -> crate::Result<String> {
@@ -69,25 +247,37 @@ fn run_unparser(expr: &Expr) -> crate::Result<String> {
 }
 
 pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
-    // Fast path: no binary literals — DataFusion's unparser handles everything.
-    if !has_binary_literal(expr) {
-        return run_unparser(expr);
-    }
-
-    // Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary`
-    // scalars, so we rewrite each one to a unique string-literal placeholder,
-    // let the unparser do the rest of the work, then substitute the SQL
-    // `X'...'` byte-string literal back in.  This keeps the operator/function
-    // serialization logic centralized in DataFusion and works for every
-    // expression node type the unparser supports.
-    let mut bindings: Vec<Vec<u8>> = Vec::new();
+    // DataFusion's unparser needs a few adaptations before its SQL can be
+    // reparsed by Lance without changing the typed expression's semantics:
+    //
+    // * decimal literals need an explicit cast to preserve precision and scale;
+    // * casts need exact Arrow type names rather than SQL type aliases;
+    // * an empty IN list is valid in DataFusion but invalid SQL;
+    // * binary literals are unsupported by the unparser and need placeholders.
+    // Eliminate empty membership expressions before visiting their children.
+    // Otherwise a discarded binary child could leave behind a stale binding.
     let rewritten = expr
         .clone()
+        .transform(|e: Expr| match e {
+            Expr::InList(in_list) if in_list.list.is_empty() => Ok(Transformed::yes(
+                Expr::Literal(ScalarValue::Boolean(Some(in_list.negated)), None),
+            )),
+            other => Ok(Transformed::no(other)),
+        })
+        .map_err(|e| crate::Error::InvalidInput {
+            message: format!("failed to rewrite expression: {e}"),
+        })?
+        .data;
+
+    let user_strings = string_literals(&rewritten);
+    let mut next_placeholder_id = 0;
+    let mut binary_bindings = HashMap::new();
+    let rewritten = rewritten
         .transform(|e: Expr| match e {
             Expr::Literal(ScalarValue::Binary(Some(bytes)), m)
             | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => {
-                let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len());
-                bindings.push(bytes);
+                let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id);
+                binary_bindings.insert(placeholder.clone(), bytes);
                 Ok(Transformed::yes(Expr::Literal(
                     ScalarValue::Utf8(Some(placeholder)),
                     m,
@@ -97,6 +287,57 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
             | Expr::Literal(ScalarValue::LargeBinary(None), m) => {
                 Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m)))
             }
+            Expr::Literal(ScalarValue::Decimal32(Some(value), precision, scale), _m) => {
+                let value = Decimal32Type::format_decimal(value, precision, scale);
+                Ok(Transformed::yes(typed_string_literal(
+                    value,
+                    DataType::Decimal32(precision, scale),
+                )))
+            }
+            Expr::Literal(ScalarValue::Decimal64(Some(value), precision, scale), _m) => {
+                let value = Decimal64Type::format_decimal(value, precision, scale);
+                Ok(Transformed::yes(typed_string_literal(
+                    value,
+                    DataType::Decimal64(precision, scale),
+                )))
+            }
+            Expr::Literal(ScalarValue::Decimal128(Some(value), precision, scale), _m) => {
+                let value = Decimal128Type::format_decimal(value, precision, scale);
+                Ok(Transformed::yes(typed_string_literal(
+                    value,
+                    DataType::Decimal128(precision, scale),
+                )))
+            }
+            Expr::Literal(ScalarValue::Decimal256(Some(value), precision, scale), _m) => {
+                let value = Decimal256Type::format_decimal(value, precision, scale);
+                Ok(Transformed::yes(typed_string_literal(
+                    value,
+                    DataType::Decimal256(precision, scale),
+                )))
+            }
+            Expr::Literal(ScalarValue::Float16(Some(value)), _m) if !value.is_finite() => Ok(
+                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float16)),
+            ),
+            Expr::Literal(ScalarValue::Float32(Some(value)), _m) if !value.is_finite() => Ok(
+                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float32)),
+            ),
+            Expr::Literal(ScalarValue::Float64(Some(value)), _m) if !value.is_finite() => Ok(
+                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float64)),
+            ),
+            Expr::Cast(cast) => Ok(Transformed::yes(datafusion_arrow_cast(
+                *cast.expr,
+                Expr::Literal(
+                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
+                    None,
+                ),
+            ))),
+            Expr::TryCast(cast) => Ok(Transformed::yes(datafusion_arrow_try_cast(
+                *cast.expr,
+                Expr::Literal(
+                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
+                    None,
+                ),
+            ))),
             other => Ok(Transformed::no(other)),
         })
         .map_err(|e| crate::Error::InvalidInput {
@@ -104,12 +345,58 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
         })?
         .data;
 
-    let mut sql = run_unparser(&rewritten)?;
-    for (i, bytes) in bindings.iter().enumerate() {
-        // The unparser quotes string literals with single quotes, so the
-        // placeholder appears as `'__lancedb_binary_placeholder_<i>__'`.
-        let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i);
-        sql = sql.replace(&quoted, &bytes_to_hex_sql(bytes));
+    let sql = run_unparser(&rewritten)?;
+    if binary_bindings.is_empty() {
+        Ok(sql)
+    } else {
+        bind_binary_literals(&sql, binary_bindings)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::canonicalize_sql_predicate;
+
+    #[test]
+    fn normalizes_double_quoted_identifiers() {
+        assert_eq!(
+            canonicalize_sql_predicate(r#""PartyAbbrev" = 'D'"#).unwrap(),
+            "`PartyAbbrev` = 'D'"
+        );
+        assert_eq!(
+            canonicalize_sql_predicate(r#""MetaData"."userId" = 5"#).unwrap(),
+            "`MetaData`.`userId` = 5"
+        );
+        assert_eq!(
+            canonicalize_sql_predicate(r#""a""b" = 1"#).unwrap(),
+            "`a\"b` = 1"
+        );
+    }
+
+    #[test]
+    fn preserves_quotes_inside_literals_and_backticks() {
+        let filter = r#"name = 'Alice "Ace"' AND `quoted"field` = 1"#;
+        assert_eq!(canonicalize_sql_predicate(filter).unwrap(), filter);
+    }
+
+    #[test]
+    fn preserves_literals_and_comments_using_lance_dialect_rules() {
+        let predicate = r#"path = '\' AND "PartyAbbrev" = 'D' -- unmatched " in comment"#;
+        assert_eq!(
+            canonicalize_sql_predicate(predicate).unwrap(),
+            r#"path = '\' AND `PartyAbbrev` = 'D' -- unmatched " in comment"#
+        );
+
+        let predicate = r#"id = 1 /* unmatched " in block comment */"#;
+        assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
+
+        let predicate = r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#;
+        assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
+    }
+
+    #[test]
+    fn rejects_unterminated_double_quoted_identifier() {
+        let error = canonicalize_sql_predicate(r#""PartyAbbrev = 'D'"#).unwrap_err();
+        assert!(matches!(error, crate::Error::InvalidInput { .. }));
     }
-    Ok(sql)
 }

rust/lancedb/src/function.rs

@@ -5,9 +5,9 @@
 //! backend-neutral terminal result of a computed-column refresh.
 //!
 //! This module contains client/wire values only. Catalog persistence,
-//! environment bake, and execution are owned by Sophon.
+//! environment bake, secret resolution, and execution are owned by Sophon.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use serde::de::{self, DeserializeOwned};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -15,6 +15,16 @@ use serde_json::Value;
 
 use crate::{Error, Result};
 
+// Keep these byte limits aligned with Sophon's Function submission validation.
+pub(crate) const MAX_FUNCTION_SECRET_VALUE_BYTES: usize = 64 * 1024;
+const MAX_FUNCTION_SECRET_VALUES_BYTES: usize = 512 * 1024;
+
+fn is_portable_environment_name(name: &str) -> bool {
+    let mut bytes = name.bytes();
+    matches!(bytes.next(), Some(b'A'..=b'Z' | b'a'..=b'z' | b'_'))
+        && bytes.all(|byte| matches!(byte, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_'))
+}
+
 fn invalid_json(error: impl std::fmt::Display) -> Error {
     Error::InvalidInput {
         message: format!("invalid remote Function JSON: {error}"),
@@ -198,6 +208,11 @@ pub struct PythonEnvironmentSpec {
 }
 
 /// Reproducible Python runtime definition understood by Sophon.
+///
+/// `env` contains non-secret values. Secret values are submission-only in the
+/// client model and do not become part of this public runtime identity;
+/// [`FunctionVersion::required_secrets`] contains names only. Sophon persists
+/// submitted values separately in the private execution artifact.
 #[derive(Debug, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum PythonRuntimeSpec {
@@ -239,7 +254,7 @@ impl PythonRuntimeSpec {
         }
     }
 
-    /// Environment variables, or `None` for an unknown kind.
+    /// Non-secret environment variables, or `None` for an unknown kind.
     pub fn env(&self) -> Option<&BTreeMap<String, String>> {
         match self {
             Self::Python { env, .. } => Some(env),
@@ -324,6 +339,8 @@ pub struct FunctionVersion {
     runtime: PythonRuntimeSpec,
     runtime_digest: String,
     environment_digest: String,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    required_secrets: Vec<String>,
     created_at: String,
 }
 
@@ -356,6 +373,12 @@ impl FunctionVersion {
         &self.environment_digest
     }
 
+    /// Required secret names. Resolved values exist only in Sophon's private
+    /// execution artifact and worker launch path.
+    pub fn required_secrets(&self) -> &[String] {
+        &self.required_secrets
+    }
+
     pub fn created_at(&self) -> &str {
         &self.created_at
     }
@@ -397,12 +420,115 @@ pub struct FunctionArtifactRequest {
 }
 
 /// Stable request envelope for remote immutable Function registration.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+///
+/// Secret values are submission-only in the client model. Sophon persists them
+/// in the database-scoped private execution artifact; returned
+/// [`FunctionVersion`] and Job metadata contain only
+/// [`Self::required_secrets`] names. Debug formatting always redacts values.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct FunctionRegistrationRequest {
     pub name: String,
     pub artifact: FunctionArtifactRequest,
     pub signature: FunctionSignature,
     pub runtime: PythonRuntimeSpec,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub required_secrets: Vec<String>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub secret_values: BTreeMap<String, String>,
+}
+
+impl FunctionRegistrationRequest {
+    pub(crate) fn validate_secret_values(&self) -> Result<()> {
+        let mut required = BTreeSet::new();
+        for name in &self.required_secrets {
+            if !is_portable_environment_name(name) {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "Function secret name {name:?} must be a portable environment variable name"
+                    ),
+                });
+            }
+            if !required.insert(name) {
+                return Err(Error::InvalidInput {
+                    message: format!("Function required_secrets contains duplicate name {name:?}"),
+                });
+            }
+        }
+
+        if let PythonRuntimeSpec::Python { env, .. } = &self.runtime
+            && let Some(name) = required.iter().find(|name| env.contains_key(**name))
+        {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "Function runtime env and secret names must be disjoint: {name:?}"
+                ),
+            });
+        }
+
+        let provided = self.secret_values.keys().collect::<BTreeSet<_>>();
+        if required != provided {
+            return Err(Error::InvalidInput {
+                message: "Function secret_values keys must exactly match required_secrets"
+                    .to_string(),
+            });
+        }
+
+        let mut total_bytes = 0usize;
+        for (name, value) in &self.secret_values {
+            if value.is_empty() {
+                return Err(Error::InvalidInput {
+                    message: format!("Function secret {name:?} value must be non-empty"),
+                });
+            }
+            if value.contains('\0') {
+                return Err(Error::InvalidInput {
+                    message: format!("Function secret {name:?} value must not contain NUL"),
+                });
+            }
+            if value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "Function secret {name:?} value exceeds the \
+                         {MAX_FUNCTION_SECRET_VALUE_BYTES}-byte limit"
+                    ),
+                });
+            }
+            total_bytes =
+                total_bytes
+                    .checked_add(value.len())
+                    .ok_or_else(|| Error::InvalidInput {
+                        message: "Function secret values exceed the request byte limit".to_string(),
+                    })?;
+        }
+        if total_bytes > MAX_FUNCTION_SECRET_VALUES_BYTES {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "Function secret values exceed the \
+                     {MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"
+                ),
+            });
+        }
+        Ok(())
+    }
+}
+
+impl std::fmt::Debug for FunctionRegistrationRequest {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let secret_values = self
+            .secret_values
+            .keys()
+            .map(|name| (name, "[REDACTED]"))
+            .collect::<BTreeMap<_, _>>();
+        formatter
+            .debug_struct("FunctionRegistrationRequest")
+            .field("name", &self.name)
+            .field("artifact", &self.artifact)
+            .field("signature", &self.signature)
+            .field("runtime", &self.runtime)
+            .field("required_secrets", &self.required_secrets)
+            .field("secret_values", &secret_values)
+            .finish()
+    }
 }
 
 impl_json!(FunctionRegistrationRequest);
@@ -587,6 +713,185 @@ impl RefreshColumnResult {
 
 impl_json!(RefreshColumnResult);
 
+#[cfg(test)]
+mod secret_value_tests {
+    use super::{
+        FunctionRegistrationRequest, MAX_FUNCTION_SECRET_VALUE_BYTES,
+        MAX_FUNCTION_SECRET_VALUES_BYTES, PythonRuntimeSpec,
+    };
+    use crate::Error;
+
+    fn request() -> FunctionRegistrationRequest {
+        FunctionRegistrationRequest::from_json(include_str!(
+            "../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
+        ))
+        .unwrap()
+    }
+
+    #[test]
+    fn validates_secret_name_and_value_invariants() {
+        let missing = request();
+        assert!(matches!(
+            missing.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("exactly match")
+        ));
+
+        let mut empty = request();
+        empty
+            .secret_values
+            .insert("API_TOKEN".to_string(), String::new());
+        assert!(matches!(
+            empty.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("non-empty")
+        ));
+
+        let mut nul = request();
+        nul.secret_values
+            .insert("API_TOKEN".to_string(), "before\0after".to_string());
+        assert!(matches!(
+            nul.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("NUL")
+        ));
+
+        let mut unexpected = request();
+        unexpected
+            .secret_values
+            .insert("OTHER".to_string(), "value".to_string());
+        assert!(matches!(
+            unexpected.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("exactly match")
+        ));
+    }
+
+    #[test]
+    fn rejects_invalid_duplicate_and_overlapping_secret_declarations() {
+        let mut invalid_name = request();
+        invalid_name.required_secrets = vec!["BAD=NAME".to_string()];
+        invalid_name
+            .secret_values
+            .insert("BAD=NAME".to_string(), "secret".to_string());
+
+        let mut duplicate = request();
+        duplicate.required_secrets = vec!["API_TOKEN".to_string(), "API_TOKEN".to_string()];
+        duplicate
+            .secret_values
+            .insert("API_TOKEN".to_string(), "secret".to_string());
+
+        let mut overlap = request();
+        overlap
+            .secret_values
+            .insert("API_TOKEN".to_string(), "secret".to_string());
+        if let PythonRuntimeSpec::Python { env, .. } = &mut overlap.runtime {
+            env.insert("API_TOKEN".to_string(), "public".to_string());
+        }
+
+        assert!(matches!(
+            invalid_name.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("portable environment variable")
+        ));
+        assert!(matches!(
+            duplicate.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("duplicate")
+        ));
+        assert!(matches!(
+            overlap.validate_secret_values(),
+            Err(Error::InvalidInput { message }) if message.contains("must be disjoint")
+        ));
+    }
+
+    #[test]
+    fn enforces_portable_secret_name_boundaries() {
+        for name in ["A", "_", "A0_"] {
+            let mut request = request();
+            request.required_secrets = vec![name.to_string()];
+            request
+                .secret_values
+                .insert(name.to_string(), "secret".to_string());
+            request.validate_secret_values().unwrap();
+        }
+
+        for name in ["", "0TOKEN", "BAD-NAME", "TÖKEN"] {
+            let mut request = request();
+            request.required_secrets = vec![name.to_string()];
+            request
+                .secret_values
+                .insert(name.to_string(), "secret".to_string());
+            assert!(matches!(
+                request.validate_secret_values(),
+                Err(Error::InvalidInput { message })
+                    if message.contains("portable environment variable")
+            ));
+        }
+    }
+
+    #[test]
+    fn accepts_exact_secret_value_utf8_byte_limit() {
+        for value in [
+            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES),
+            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len()),
+        ] {
+            assert_eq!(value.len(), MAX_FUNCTION_SECRET_VALUE_BYTES);
+            let mut request = request();
+            request.secret_values.insert("API_TOKEN".to_string(), value);
+            request.validate_secret_values().unwrap();
+        }
+    }
+
+    #[test]
+    fn rejects_secret_value_over_utf8_byte_limit() {
+        for value in [
+            "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
+            "é".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES / "é".len() + 1),
+        ] {
+            assert!(value.len() > MAX_FUNCTION_SECRET_VALUE_BYTES);
+            let mut request = request();
+            request.secret_values.insert("API_TOKEN".to_string(), value);
+            assert!(matches!(
+                request.validate_secret_values(),
+                Err(Error::InvalidInput { message }) if message.contains("65536-byte limit")
+            ));
+        }
+    }
+
+    #[test]
+    fn rejects_aggregate_secret_value_bytes_over_server_limit() {
+        let mut request = request();
+        request.required_secrets = (0..9).map(|index| format!("SECRET_{index}")).collect();
+        request.secret_values = request
+            .required_secrets
+            .iter()
+            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
+            .collect();
+
+        assert!(matches!(
+            request.validate_secret_values(),
+            Err(Error::InvalidInput { message })
+                if message.contains(&format!("{MAX_FUNCTION_SECRET_VALUES_BYTES}-byte request limit"))
+        ));
+    }
+
+    #[test]
+    fn accepts_exact_aggregate_secret_value_byte_limit() {
+        let mut request = request();
+        request.required_secrets = (0..8).map(|index| format!("SECRET_{index}")).collect();
+        request.secret_values = request
+            .required_secrets
+            .iter()
+            .map(|name| (name.clone(), "x".repeat(MAX_FUNCTION_SECRET_VALUE_BYTES)))
+            .collect();
+
+        assert_eq!(
+            request
+                .secret_values
+                .values()
+                .map(String::len)
+                .sum::<usize>(),
+            MAX_FUNCTION_SECRET_VALUES_BYTES
+        );
+        request.validate_secret_values().unwrap();
+    }
+}
+
 #[cfg(test)]
 mod conda_environment_tests {
     use super::PythonEnvironmentSpec;

rust/lancedb/src/materialized_view.rs

@@ -170,6 +170,15 @@ pub(crate) fn plan(
     filter: Option<&str>,
     limit: Option<u64>,
 ) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
+    let filter = filter
+        .map(crate::expr::canonicalize_sql_predicate)
+        .transpose()
+        .map_err(|err| match err {
+            Error::InvalidInput { message } => Error::InvalidInput {
+                message: format!("invalid view filter: {message}"),
+            },
+            err => err,
+        })?;
     let projections: Vec<(String, String)> = if projections.is_empty() {
         source_schema
             .fields()
@@ -274,7 +283,7 @@ pub(crate) fn plan(
         declared.push(output);
     }
 
-    if let Some(filter) = filter {
+    if let Some(filter) = filter.as_deref() {
         let expr = planner
             .parse_filter(filter)
             .map_err(|e| Error::InvalidInput {
@@ -314,7 +323,7 @@ pub(crate) fn plan(
             .into_iter()
             .map(|(output, expression)| ViewProjection { output, expression })
             .collect(),
-        filter: filter.map(String::from),
+        filter,
         limit,
         inputs,
     };

rust/lancedb/src/materialized_view/refresh.rs

@@ -46,8 +46,9 @@ use lance_table::format::Fragment;
 use serde::{Deserialize, Serialize};
 
 use super::{
-    INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
-    SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
+    DEFINITION_META_KEY, INCARNATION_META_KEY, MaterializedViewDefinition,
+    REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
+    definition_to_metadata,
 };
 use crate::database::OpenTableRequest;
 use crate::table::{NativeTable, NativeTableExt, Table};
@@ -197,8 +198,28 @@ pub(crate) async fn execute_refresh(
             ),
         });
     }
+    let definition_changed =
+        definition.filter != replanned.filter || definition.inputs != replanned.inputs;
     let definition = &replanned;
 
+    // A watermark written for a legacy raw filter certifies the rows that
+    // filter produced, not the canonical predicate above. Rebuild instead of
+    // accepting or advancing it, and persist the migrated definition in the
+    // same metadata commit that certifies the replacement rows.
+    if definition_changed {
+        return rebuild(
+            view_native,
+            &view_ds,
+            &source_ds,
+            source_version,
+            source_ts,
+            definition,
+            true,
+            expected_incarnation,
+        )
+        .await;
+    }
+
     let metadata = &view_ds.schema().metadata;
     let watermark: Option<u64> = metadata
         .get(SOURCE_VERSION_META_KEY)
@@ -257,6 +278,7 @@ pub(crate) async fn execute_refresh(
                         source_version,
                         source_ts,
                         definition,
+                        false,
                         expected_incarnation,
                     )
                     .await
@@ -271,6 +293,7 @@ pub(crate) async fn execute_refresh(
                 source_version,
                 source_ts,
                 definition,
+                false,
                 expected_incarnation,
             )
             .await
@@ -683,6 +706,7 @@ async fn incremental(
             view_ds.clone(),
             source_version,
             source_ts,
+            None,
             expected_incarnation,
         )
         .await?;
@@ -704,6 +728,7 @@ async fn incremental(
             published,
             source_version,
             source_ts,
+            None,
             expected_incarnation,
         )
         .await?;
@@ -775,6 +800,7 @@ async fn incremental(
             published,
             source_version,
             source_ts,
+            None,
             expected_incarnation,
         )
         .await?;
@@ -824,12 +850,14 @@ async fn incremental(
         appended,
         source_version,
         source_ts,
+        None,
         expected_incarnation,
     )
     .await?;
     Ok(Some(result))
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn rebuild(
     view_native: &NativeTable,
     view_ds: &Dataset,
@@ -837,6 +865,7 @@ async fn rebuild(
     source_version: u64,
     source_ts: u128,
     definition: &MaterializedViewDefinition,
+    persist_definition: bool,
     expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
     let rows_written = Arc::new(AtomicU64::new(0));
@@ -867,6 +896,7 @@ async fn rebuild(
         replaced,
         source_version,
         source_ts,
+        persist_definition.then_some(definition),
         expected_incarnation,
     )
     .await?;
@@ -981,6 +1011,7 @@ async fn stamp_watermark(
     mut dataset: Dataset,
     source_version: u64,
     source_ts: u128,
+    definition: Option<&MaterializedViewDefinition>,
     expected_incarnation: Option<&str>,
 ) -> Result<u64> {
     ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
@@ -993,27 +1024,32 @@ async fn stamp_watermark(
         .get(INCARNATION_META_KEY)
         .cloned()
         .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
-    dataset
-        .update_schema_metadata([
-            (INCARNATION_META_KEY.to_string(), Some(incarnation)),
-            (
-                SOURCE_VERSION_META_KEY.to_string(),
-                Some(source_version.to_string()),
-            ),
-            (
-                SOURCE_VERSION_TS_META_KEY.to_string(),
-                Some(source_ts.to_string()),
-            ),
-            (
-                REFRESHED_AT_MS_META_KEY.to_string(),
-                Some(now_ms().to_string()),
-            ),
-            (
-                VIEW_VERSION_META_KEY.to_string(),
-                Some(predicted.to_string()),
-            ),
-        ])
-        .await?;
+    let mut metadata = vec![(INCARNATION_META_KEY.to_string(), Some(incarnation))];
+    if let Some(definition) = definition {
+        metadata.push((
+            DEFINITION_META_KEY.to_string(),
+            Some(definition_to_metadata(definition)?),
+        ));
+    }
+    metadata.extend([
+        (
+            SOURCE_VERSION_META_KEY.to_string(),
+            Some(source_version.to_string()),
+        ),
+        (
+            SOURCE_VERSION_TS_META_KEY.to_string(),
+            Some(source_ts.to_string()),
+        ),
+        (
+            REFRESHED_AT_MS_META_KEY.to_string(),
+            Some(now_ms().to_string()),
+        ),
+        (
+            VIEW_VERSION_META_KEY.to_string(),
+            Some(predicted.to_string()),
+        ),
+    ]);
+    dataset.update_schema_metadata(metadata).await?;
     let actual = dataset.version().version;
     if actual != predicted {
         return Err(Error::Runtime {
@@ -1585,6 +1621,106 @@ mod tests {
         assert_eq!(read(view.table(), "x").await, vec![20, 40]);
     }
 
+    #[tokio::test]
+    async fn test_mixed_case_filter_is_canonicalized_for_lineage_and_refresh() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(
+            ("id", Int32, [1, 2, 3]),
+            ("PartyAbbrev", Utf8, ["D", "R", "D"])
+        )
+        .unwrap();
+        conn.create_table("src", batch)
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        conn.create_materialized_view("democrats", "src")
+            .select([("id", "id")])
+            .only_if(r#""PartyAbbrev" = 'D'"#)
+            .execute()
+            .await
+            .unwrap();
+
+        // Reopen from schema metadata so these assertions cover the stored
+        // predicate and lineage, not only the declaration-time handle.
+        let view = conn.open_materialized_view("democrats").await.unwrap();
+        assert_eq!(
+            view.definition().filter.as_deref(),
+            Some("`PartyAbbrev` = 'D'")
+        );
+        assert_eq!(view.definition().inputs, ["PartyAbbrev", "id"]);
+
+        let result = view.refresh().execute().await.unwrap();
+        assert_eq!(result.rows_written, 2);
+        assert_eq!(read(view.table(), "id").await, vec![1, 3]);
+    }
+
+    #[tokio::test]
+    async fn test_legacy_raw_filter_rebuilds_and_persists_canonical_definition() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let batch = record_batch!(
+            ("id", Int32, [1, 2, 3]),
+            ("PartyAbbrev", Utf8, ["D", "R", "D"])
+        )
+        .unwrap();
+        conn.create_table("legacy_src", batch)
+            .write_options(crate::materialized_view::tests::stable_row_ids())
+            .execute()
+            .await
+            .unwrap();
+        let view = conn
+            .create_materialized_view("legacy_view", "legacy_src")
+            .select([("id", "id")])
+            .only_if(r#""PartyAbbrev" = 'X'"#)
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(view.refresh().execute().await.unwrap().rows_written, 0);
+
+        // Model a definition and up-to-date watermark written before filter
+        // canonicalization was applied to materialized views.
+        let mut legacy = view.definition().clone();
+        legacy.filter = Some(r#""PartyAbbrev" = 'D'"#.into());
+        legacy.inputs = vec!["id".into()];
+        let native = view.table().as_native().unwrap();
+        let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
+        let predicted = dataset.version().version + 1;
+        dataset
+            .update_schema_metadata([
+                (
+                    DEFINITION_META_KEY.to_string(),
+                    Some(definition_to_metadata(&legacy).unwrap()),
+                ),
+                (
+                    VIEW_VERSION_META_KEY.to_string(),
+                    Some(predicted.to_string()),
+                ),
+            ])
+            .await
+            .unwrap();
+        native.dataset.update(dataset);
+
+        let reopened = conn.open_materialized_view("legacy_view").await.unwrap();
+        let result = reopened.refresh().execute().await.unwrap();
+        assert_eq!(result.mode, RefreshMode::Rebuild);
+        assert_eq!(result.rows_written, 2);
+        assert_eq!(read(reopened.table(), "id").await, vec![1, 3]);
+
+        // A fresh handle proves the migration was stored alongside the new
+        // watermark and therefore happens only once.
+        let migrated = conn.open_materialized_view("legacy_view").await.unwrap();
+        assert_eq!(
+            migrated.definition().filter.as_deref(),
+            Some("`PartyAbbrev` = 'D'")
+        );
+        assert_eq!(migrated.definition().inputs, ["PartyAbbrev", "id"]);
+        assert_eq!(
+            migrated.refresh().execute().await.unwrap().mode,
+            RefreshMode::NoOp
+        );
+        assert_eq!(read(migrated.table(), "id").await, vec![1, 3]);
+    }
+
     #[tokio::test]
     async fn test_append_refreshes_incrementally() {
         let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
@@ -2767,7 +2903,7 @@ mod tests {
         let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
         view.table().delete("x = 1").await.unwrap();
 
-        let err = stamp_watermark(view_native, stale, 99, 99, None).await;
+        let err = stamp_watermark(view_native, stale, 99, 99, None, None).await;
         assert!(err.is_err());
 
         let result = view.refresh().execute().await.unwrap();

rust/lancedb/src/query.rs

@@ -399,6 +399,9 @@ pub trait QueryBase {
     /// x > 5 OR y = 'test'
     /// ```
     ///
+    /// Identifiers may be delimited with SQL-standard double quotes or
+    /// backticks. String literals must use single quotes.
+    ///
     /// Filtering performance can often be improved by creating a scalar index
     /// on the filter column(s).
     ///
@@ -913,6 +916,17 @@ impl QueryRequest {
     /// use different representations) the error is recorded and surfaced later
     /// by [`Self::check_filter`].
     pub(crate) fn add_filter(&mut self, new: QueryFilter) {
+        let new = match new {
+            QueryFilter::Sql(filter) => match crate::expr::canonicalize_sql_predicate(&filter) {
+                Ok(filter) => QueryFilter::Sql(filter),
+                Err(err) => {
+                    self.filter_error = Some(err.to_string());
+                    return;
+                }
+            },
+            other => other,
+        };
+
         self.filter = Some(match self.filter.take() {
             None => new,
             Some(existing) => match and_filters(existing, new) {
@@ -1174,12 +1188,12 @@ impl VectorQuery {
 
     /// Add another query vector to the search.
     ///
-    /// Multiple searches will be dispatched as part of the query.
-    /// This is a convenience method for adding multiple query vectors
-    /// to the search. It is not expected to be faster than issuing
-    /// multiple queries concurrently.
+    /// Multiple searches will be dispatched as a batch. Flat searches share
+    /// one table scan across the query vectors, avoiding the scan and memory
+    /// amplification of issuing the searches concurrently. Indexed searches
+    /// may still perform per-vector index work.
     ///
-    /// The output data will contain an additional columns `query_index` which
+    /// The output data will contain an additional column `query_index` which
     /// will contain the index of the query vector that was used to generate the
     /// result.
     pub fn add_query_vector(mut self, vector: impl IntoQueryVector) -> Result<Self> {
@@ -1646,10 +1660,14 @@ mod tests {
     use std::{collections::HashSet, sync::Arc};
 
     use super::*;
-    use arrow::{array::downcast_array, compute::concat_batches, datatypes::Int32Type};
+    use arrow::{
+        array::downcast_array,
+        compute::concat_batches,
+        datatypes::{Int32Type, UInt8Type},
+    };
     use arrow_array::{
-        FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, cast::AsArray,
-        types::Float32Type,
+        FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
+        StringArray, cast::AsArray, types::Float32Type,
     };
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
     use futures::{StreamExt, TryStreamExt};
@@ -1878,6 +1896,157 @@ mod tests {
         query.execute().await.unwrap();
     }
 
+    #[tokio::test]
+    async fn test_double_quoted_predicates_across_table_operations() {
+        let tmp_dir = tempdir().unwrap();
+        let dataset_path = tmp_dir.path().join("test.lance");
+        let uri = dataset_path.to_str().unwrap();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("PartyAbbrev", DataType::Utf8, false),
+            ArrowField::new("path", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
+                Arc::new(StringArray::from(vec!["D", "R", "R", "D"])),
+                Arc::new(StringArray::from(vec!["\\", "\\", "x", "x"])),
+            ],
+        )
+        .unwrap();
+
+        let conn = connect(uri).execute().await.unwrap();
+        let table = conn.create_table("parties", batch).execute().await.unwrap();
+        let batches = table
+            .query()
+            .only_if(r#""PartyAbbrev" = 'D'"#)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
+        assert_eq!(
+            table
+                .count_rows(Some(r#""PartyAbbrev" = 'D'"#.to_string()))
+                .await
+                .unwrap(),
+            2
+        );
+
+        // Public BaseTable dispatch cannot bypass canonicalization.
+        let query = AnyQuery::Query(QueryRequest {
+            filter: Some(QueryFilter::Sql(r#""PartyAbbrev" = 'D'"#.to_string())),
+            ..Default::default()
+        });
+        let batches = table
+            .base_table()
+            .query(&query, Default::default())
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
+        assert_eq!(
+            table
+                .base_table()
+                .count_rows(Some(crate::table::Filter::Sql(
+                    r#""PartyAbbrev" = 'D'"#.to_string(),
+                )))
+                .await
+                .unwrap(),
+            2
+        );
+
+        for predicate in [
+            r#"id = 1 -- unmatched " in a valid SQL comment"#,
+            r#"id = 1 /* unmatched " in a valid SQL comment */"#,
+            r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#,
+            r#"path = '\' AND "PartyAbbrev" = 'D'"#,
+        ] {
+            let batches = table
+                .query()
+                .only_if(predicate)
+                .execute()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
+        }
+
+        // The same canonical predicate contract applies to both merge filters.
+        let source = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["D", "R", "R"])),
+                Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
+            ],
+        )
+        .unwrap();
+        let mut merge = table.merge_insert(&["id"]);
+        merge.when_not_matched_by_source_delete(Some(r#""PartyAbbrev" = 'D'"#.to_string()));
+        let result = table
+            .base_table()
+            .merge_insert(
+                merge,
+                Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone())),
+            )
+            .await
+            .unwrap();
+        assert_eq!(result.num_deleted_rows, 1);
+
+        let source = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["U", "U", "U"])),
+                Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
+            ],
+        )
+        .unwrap();
+        let mut merge = table.merge_insert(&["id"]);
+        merge.when_matched_update_all(Some(r#"target."PartyAbbrev" = 'D'"#.to_string()));
+        merge
+            .execute(Box::new(RecordBatchIterator::new(vec![Ok(source)], schema)))
+            .await
+            .unwrap();
+        assert_eq!(
+            table
+                .count_rows(Some(r#""PartyAbbrev" = 'U'"#.to_string()))
+                .await
+                .unwrap(),
+            1
+        );
+
+        let update = table
+            .update()
+            .only_if(r#""PartyAbbrev" = 'R'"#)
+            .column("PartyAbbrev", "'X'");
+        table.base_table().update(update).await.unwrap();
+        assert_eq!(
+            table
+                .count_rows(Some(r#""PartyAbbrev" = 'X'"#.to_string()))
+                .await
+                .unwrap(),
+            2
+        );
+
+        let result = table
+            .base_table()
+            .delete(crate::table::Predicate::String(r#""PartyAbbrev" = 'X'"#))
+            .await
+            .unwrap();
+        assert_eq!(result.num_deleted_rows, 2);
+        assert_eq!(table.count_rows(None).await.unwrap(), 1);
+    }
+
     #[tokio::test]
     async fn test_select_with_transform() {
         let batches = make_non_empty_batches();
@@ -2334,7 +2503,8 @@ mod tests {
             .limit(1);
 
         let plan = query.explain_plan(true).await.unwrap();
-        assert!(plan.contains("UnionExec"));
+        assert!(plan.contains("KNNVectorDistance: queries=2"));
+        assert!(!plan.contains("UnionExec"));
 
         let results = query
             .execute()
@@ -2349,6 +2519,100 @@ mod tests {
         // We don't guarantee order.
         assert!(query_index.values().contains(&0));
         assert!(query_index.values().contains(&1));
+
+        // Batch KNN does not support a per-query offset, so offset queries keep
+        // the legacy per-vector plan to preserve their result semantics.
+        let offset_query = table
+            .query()
+            .nearest_to(&[0.1, 0.2, 0.3, 0.4])
+            .unwrap()
+            .add_query_vector(&[0.5, 0.6, 0.7, 0.8])
+            .unwrap()
+            .limit(1)
+            .offset(1);
+        assert!(
+            offset_query
+                .explain_plan(true)
+                .await
+                .unwrap()
+                .contains("UnionExec")
+        );
+        let offset_results = offset_query
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(
+            offset_results
+                .iter()
+                .map(RecordBatch::num_rows)
+                .sum::<usize>(),
+            2
+        );
+    }
+
+    #[tokio::test]
+    async fn test_multiple_binary_query_vectors() {
+        let vectors = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
+            vec![
+                Some(vec![Some(0), Some(0)]),
+                Some(vec![Some(255), Some(255)]),
+            ],
+            2,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("vector", vectors.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(vectors)],
+        )
+        .unwrap();
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let table = conn
+            .create_table("binary_batch", batch)
+            .execute()
+            .await
+            .unwrap();
+        let query = table
+            .query()
+            .nearest_to(&[0.0, 0.0])
+            .unwrap()
+            .add_query_vector(&[255.0, 255.0])
+            .unwrap()
+            .distance_type(DistanceType::Hamming)
+            .limit(1);
+
+        // Binary queries retain the per-vector plan because Lance's binary
+        // nearest path requires primitive UInt8 query arrays.
+        assert!(
+            query
+                .explain_plan(true)
+                .await
+                .unwrap()
+                .contains("UnionExec")
+        );
+
+        let results = query
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let results = concat_batches(&results[0].schema(), &results).unwrap();
+        assert_eq!(results.num_rows(), 2);
+
+        let ids = results["id"].as_primitive::<Int32Type>();
+        assert!(ids.values().contains(&0));
+        assert!(ids.values().contains(&1));
+        let query_index = results["query_index"].as_primitive::<Int32Type>();
+        assert!(query_index.values().contains(&0));
+        assert!(query_index.values().contains(&1));
     }
 
     #[tokio::test]

rust/lancedb/src/remote/client.rs

@@ -7,6 +7,7 @@ use reqwest::{
     Body, Request, RequestBuilder, Response,
     header::{HeaderMap, HeaderValue},
 };
+use serde_json::Value;
 use std::{collections::HashMap, future::Future, str::FromStr, sync::Arc, time::Duration};
 
 use crate::error::{Error, Result};
@@ -14,6 +15,60 @@ use crate::remote::db::RemoteOptions;
 use crate::remote::retry::{ResolvedRetryConfig, RetryCounter};
 
 const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
+const REDACTED_JSON_VALUE: &str = "[REDACTED]";
+const SUPPRESSED_JSON_BODY: &str = "[JSON BODY SUPPRESSED]";
+
+fn is_sensitive_json_field(name: &str) -> bool {
+    name.to_ascii_lowercase().contains("secret")
+}
+
+fn redact_sensitive_json_fields(value: &mut Value) {
+    match value {
+        Value::Object(fields) => {
+            for (name, child) in fields {
+                if is_sensitive_json_field(name) {
+                    *child = Value::String(REDACTED_JSON_VALUE.to_string());
+                } else {
+                    redact_sensitive_json_fields(child);
+                }
+            }
+        }
+        Value::Array(values) => values.iter_mut().for_each(redact_sensitive_json_fields),
+        _ => {}
+    }
+}
+
+fn redacted_json_body(request: &Request) -> Option<String> {
+    let body = request.body()?.as_bytes()?;
+    let mut value = serde_json::from_slice(body).ok()?;
+    redact_sensitive_json_fields(&mut value);
+    serde_json::to_string(&value).ok()
+}
+
+fn request_log_message(request: &Request, request_id: &str) -> String {
+    let prefix = format!(
+        "Sending request_id={}: {} {}",
+        request_id,
+        request.method(),
+        request.url()
+    );
+    let content_type = request
+        .headers()
+        .get("content-type")
+        .and_then(|value| value.to_str().ok())
+        .and_then(|value| value.split(';').next());
+    if content_type.is_some_and(|value| value.eq_ignore_ascii_case("application/json")) {
+        // Never format the raw Request here: its Debug representation is not a
+        // redaction boundary and may include the original body. If the JSON body
+        // cannot be structurally parsed, suppress it instead of logging raw bytes.
+        let body = redacted_json_body(request).unwrap_or_else(|| SUPPRESSED_JSON_BODY.to_string());
+        format!("{prefix} with body {body}")
+    } else {
+        // Method and URL are sufficient request context. Raw Request formatting
+        // may expose headers or a non-JSON body, so it is never a logging fallback.
+        prefix
+    }
+}
 
 /// Configuration for TLS/mTLS settings.
 #[derive(Clone, Debug)]
@@ -839,22 +894,9 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
         }
     }
 
-    pub(crate) fn log_request(&self, request: &Request, request_id: &String) {
+    pub(crate) fn log_request(&self, request: &Request, request_id: &str) {
         if log::log_enabled!(log::Level::Debug) {
-            let content_type = request
-                .headers()
-                .get("content-type")
-                .map(|v| v.to_str().unwrap());
-            if content_type == Some("application/json") {
-                let body = request.body().as_ref().unwrap().as_bytes().unwrap();
-                let body = String::from_utf8_lossy(body);
-                debug!(
-                    "Sending request_id={}: {:?} with body {}",
-                    request_id, request, body
-                );
-            } else {
-                debug!("Sending request_id={}: {:?}", request_id, request);
-            }
+            debug!("{}", request_log_message(request, request_id));
         }
     }
 
@@ -1077,6 +1119,49 @@ mod tests {
         ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner())
     }
 
+    #[test]
+    fn test_request_log_message_redacts_secrets_and_never_formats_raw_requests() {
+        const SECRET_SENTINEL: &str = "udf-secret-log-sentinel-7e4e";
+        const MALFORMED_SENTINEL: &str = "malformed-secret-log-sentinel-b652";
+        const NON_JSON_SENTINEL: &str = "non-json-secret-log-sentinel-7fd1";
+
+        let request = reqwest::Client::new()
+            .post("https://example.com/v1/functions/create")
+            .json(&serde_json::json!({
+                "name": "uses_secret",
+                "nested": {
+                    "secret_values": {"OPENAI_API_KEY": SECRET_SENTINEL},
+                    "safe": "visible-value"
+                }
+            }))
+            .build()
+            .unwrap();
+        let log_message = request_log_message(&request, "valid-json");
+
+        let malformed_request = reqwest::Client::new()
+            .post("https://example.com/v1/functions/create")
+            .header("content-type", "application/json; charset=utf-8")
+            .body(format!(r#"{{"secret_values":"{MALFORMED_SENTINEL}""#))
+            .build()
+            .unwrap();
+        let malformed_log_message = request_log_message(&malformed_request, "malformed-json");
+
+        let non_json_request = reqwest::Client::new()
+            .post("https://example.com/v1/functions/create")
+            .header("content-type", "text/plain")
+            .body(NON_JSON_SENTINEL)
+            .build()
+            .unwrap();
+        let non_json_log_message = request_log_message(&non_json_request, "non-json");
+
+        assert!(log_message.contains("visible-value"));
+        assert!(log_message.contains(REDACTED_JSON_VALUE));
+        assert!(!log_message.contains(SECRET_SENTINEL));
+        assert!(malformed_log_message.contains(SUPPRESSED_JSON_BODY));
+        assert!(!malformed_log_message.contains(MALFORMED_SENTINEL));
+        assert!(!non_json_log_message.contains(NON_JSON_SENTINEL));
+    }
+
     #[test]
     fn test_timeout_config_default() {
         let config = TimeoutConfig::default();

rust/lancedb/src/remote/db.rs

@@ -554,6 +554,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
         &self,
         request: FunctionRegistrationRequest,
     ) -> Result<Job<FunctionVersion>> {
+        request.validate_secret_values()?;
         let req = self.client.post("/v1/functions/create").json(&request);
         let (request_id, response) = self.client.send(req).await?;
         let response = self.client.check_response(&request_id, response).await?;
@@ -2642,7 +2643,8 @@ mod tests {
         );
         const FUNCTION_JOB: &str =
             include_str!("../../tests/fixtures/first_class_functions/v1/remote_function_job.json");
-        let expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
+        let mut expected: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
+        expected["secret_values"] = serde_json::json!({"API_TOKEN": "secret-value"});
         let conn = Connection::new_with_handler(move |request| match request.url().path() {
             "/v1/functions/create" => {
                 assert_eq!(request.method(), &reqwest::Method::POST);
@@ -2660,7 +2662,10 @@ mod tests {
                 .unwrap(),
             path => panic!("unexpected path: {path}"),
         });
-        let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
+        let mut request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
+        request
+            .secret_values
+            .insert("API_TOKEN".to_string(), "secret-value".to_string());
         let job = conn.create_function_async(request).await.unwrap();
         assert_eq!(job.id(), Some("job-function-1"));
         let version = job.wait().await.unwrap();
@@ -2668,6 +2673,32 @@ mod tests {
         assert_eq!(version.version(), "fv_01K3EXACT");
     }
 
+    #[tokio::test]
+    async fn test_create_function_async_validates_secrets_before_serialization_and_send() {
+        const REQUEST: &str = include_str!(
+            "../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
+        );
+        let sends = Arc::new(AtomicUsize::new(0));
+        let sends_ref = sends.clone();
+        let conn = Connection::new_with_handler(move |_| {
+            sends_ref.fetch_add(1, Ordering::SeqCst);
+            http::Response::builder().status(500).body("").unwrap()
+        });
+        let mut request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
+        request.secret_values.insert(
+            "API_TOKEN".to_string(),
+            "x".repeat(crate::function::MAX_FUNCTION_SECRET_VALUE_BYTES + 1),
+        );
+
+        let error = conn.create_function_async(request).await.unwrap_err();
+
+        assert!(matches!(
+            error,
+            Error::InvalidInput { message } if message.contains("65536-byte limit")
+        ));
+        assert_eq!(sends.load(Ordering::SeqCst), 0);
+    }
+
     #[tokio::test]
     async fn test_get_function_requires_and_sends_exact_version() {
         const VERSION: &str = include_str!(

rust/lancedb/src/remote/table.rs

@@ -1379,10 +1379,11 @@ impl<S: HttpSend> RemoteTable<S> {
         query: &AnyQuery,
         version: Option<u64>,
     ) -> Result<Vec<serde_json::Value>> {
+        let query = query.canonicalized()?;
         let mut base_body = serde_json::json!({ "version": version });
         self.apply_branch_body(&mut base_body);
 
-        match query {
+        match &query {
             AnyQuery::Query(query) => {
                 let mut body = base_body.clone();
                 self.apply_query_params(&mut body, query)?;
@@ -2491,7 +2492,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
         let mut body = if let Some(filter) = filter {
             let filter_sql = match filter {
-                Filter::Sql(sql) => sql.clone(),
+                Filter::Sql(sql) => crate::expr::canonicalize_sql_predicate(&sql)?,
                 Filter::Datafusion(expr) => expr_to_sql_string(&expr)?,
             };
             serde_json::json!({ "predicate": filter_sql, "version": read_snapshot.version })
@@ -2747,7 +2748,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         Ok(final_analyze)
     }
 
-    async fn update(&self, update: UpdateBuilder) -> Result<UpdateResult> {
+    async fn update(&self, mut update: UpdateBuilder) -> Result<UpdateResult> {
+        update.canonicalize_filter()?;
         self.check_mutable().await?;
         let request = self
             .client
@@ -2794,7 +2796,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     async fn delete(&self, predicate: Predicate<'_>) -> Result<DeleteResult> {
         self.check_mutable().await?;
         let predicate_sql = match predicate {
-            Predicate::String(s) => s.to_string(),
+            Predicate::String(s) => crate::expr::canonicalize_sql_predicate(s)?,
             Predicate::Expr(expr) => expr_to_sql_string(expr)?,
         };
         let mut body = serde_json::json!({ "predicate": predicate_sql });
@@ -2851,9 +2853,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
     async fn merge_insert(
         &self,
-        params: MergeInsertBuilder,
+        mut params: MergeInsertBuilder,
         new_data: Box<dyn RecordBatchReader + Send>,
     ) -> Result<MergeResult> {
+        params.canonicalize_filters()?;
         self.check_mutable().await?;
 
         let timeout = params.timeout;
@@ -3864,13 +3867,17 @@ mod tests {
             );
             assert_eq!(
                 request.body().unwrap().as_bytes().unwrap(),
-                br#"{"predicate":"a > 10","version":null}"#
+                br#"{"predicate":"`A` > 10","version":null}"#
             );
 
             http::Response::builder().status(200).body("42").unwrap()
         });
 
-        let count = table.count_rows(Some("a > 10".into())).await.unwrap();
+        let count = table
+            .base_table()
+            .count_rows(Some(Filter::Sql(r#""A" > 10"#.into())))
+            .await
+            .unwrap();
         assert_eq!(count, 42);
     }
 
@@ -4353,7 +4360,7 @@ mod tests {
                     assert_eq!(expression, "b - 1");
 
                     let only_if = value.get("predicate").unwrap().as_str().unwrap();
-                    assert_eq!(only_if, "b > 10");
+                    assert_eq!(only_if, "`B` > 10");
                 }
 
                 if old_server {
@@ -4369,14 +4376,12 @@ mod tests {
             }
         });
 
-        let result = table
+        let update = table
             .update()
             .column("a", "a + 1")
             .column("b", "b - 1")
-            .only_if("b > 10")
-            .execute()
-            .await
-            .unwrap();
+            .only_if(r#""B" > 10"#);
+        let result = table.base_table().update(update).await.unwrap();
 
         assert_eq!(result.version, if old_server { 0 } else { 43 });
         assert_eq!(result.rows_updated, if old_server { 0 } else { 5 });
@@ -4463,10 +4468,10 @@ mod tests {
 
                 let params = request.url().query_pairs().collect::<HashMap<_, _>>();
                 assert_eq!(params["on"], "some_col");
-                assert_eq!(params["when_matched_update_all"], "false");
+                assert_eq!(params["when_matched_update_all"], "true");
                 assert_eq!(params["when_not_matched_insert_all"], "false");
                 assert_eq!(params["when_not_matched_by_source_delete"], "false");
-                assert!(!params.contains_key("when_matched_update_all_filt"));
+                assert_eq!(params["when_matched_update_all_filt"], "target.`A` > 0");
                 assert!(!params.contains_key("when_not_matched_by_source_delete_filt"));
                 assert!(!params.contains_key("use_index"));
 
@@ -4483,11 +4488,9 @@ mod tests {
             }
         });
 
-        let result = table
-            .merge_insert(&["some_col"])
-            .execute(data)
-            .await
-            .unwrap();
+        let mut merge = table.merge_insert(&["some_col"]);
+        merge.when_matched_update_all(Some(r#"target."A" > 0"#.into()));
+        let result = table.base_table().merge_insert(merge, data).await.unwrap();
 
         assert_eq!(result.version, if old_server { 0 } else { 43 });
         if !old_server {
@@ -4549,7 +4552,7 @@ mod tests {
                 let body = request.body().unwrap().as_bytes().unwrap();
                 let body: serde_json::Value = serde_json::from_slice(body).unwrap();
                 let predicate = body.get("predicate").unwrap().as_str().unwrap();
-                assert_eq!(predicate, "id in (1, 2, 3)");
+                assert_eq!(predicate, "`ID` in (1, 2, 3)");
 
                 if old_server {
                     http::Response::builder()
@@ -4567,7 +4570,11 @@ mod tests {
             }
         });
 
-        let result = table.delete("id in (1, 2, 3)").await.unwrap();
+        let result = table
+            .base_table()
+            .delete(Predicate::String(r#""ID" in (1, 2, 3)"#))
+            .await
+            .unwrap();
         assert_eq!(result.version, if old_server { 0 } else { 43 });
     }
 
@@ -4659,6 +4666,7 @@ mod tests {
             let body = request.body().unwrap().as_bytes().unwrap();
             let body: serde_json::Value = serde_json::from_slice(body).unwrap();
             let expected_body = serde_json::json!({
+                "filter": "`A` > 0",
                 "k": isize::MAX as usize,
                 "prefilter": true,
                 "vector": [], // Empty vector means no vector query.
@@ -4674,9 +4682,13 @@ mod tests {
                 .unwrap()
         });
 
+        let query = AnyQuery::Query(QueryRequest {
+            filter: Some(QueryFilter::Sql(r#""A" > 0"#.into())),
+            ..Default::default()
+        });
         let data = table
-            .query()
-            .execute()
+            .base_table()
+            .query(&query, Default::default())
             .await
             .unwrap()
             .collect::<Vec<_>>()

rust/lancedb/src/table.rs

@@ -1164,7 +1164,10 @@ impl Table {
     ///
     /// * `filter` if present, only count rows matching the filter
     pub async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
-        self.inner.count_rows(filter.map(Filter::Sql)).await
+        let filter = filter
+            .map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate).map(Filter::Sql))
+            .transpose()?;
+        self.inner.count_rows(filter).await
     }
 
     /// Names of the blob v2 columns in this table, in declaration order.
@@ -1364,7 +1367,13 @@ impl Table {
     /// # });
     /// ```
     pub async fn delete(&self, predicate: impl Into<Predicate<'_>>) -> Result<DeleteResult> {
-        self.inner.delete(predicate.into()).await
+        match predicate.into() {
+            Predicate::String(predicate) => {
+                let predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
+                self.inner.delete(Predicate::String(&predicate)).await
+            }
+            predicate @ Predicate::Expr(_) => self.inner.delete(predicate).await,
+        }
     }
 
     /// Create an index on the provided column(s).
@@ -1777,7 +1786,23 @@ impl Table {
         self.inner.alter_columns(alterations).await
     }
 
-    /// Update per-field metadata (merges by default).
+    /// Update per-field (column) metadata.
+    ///
+    /// Each [`FieldMetadataUpdate`] is merged into the field's existing metadata
+    /// by default; use [`FieldMetadataUpdate::remove`] to delete a key, or
+    /// [`FieldMetadataUpdate::replace`] to swap the field's entire metadata map.
+    ///
+    /// The following keys are treated specially, by convention, and should be
+    /// used when appropriate:
+    ///
+    /// - `lancedb:description`: for a human-readable description of a field.
+    /// - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
+    ///   names the tag category; e.g. `lancedb:tag:model: "clip"`.
+    /// - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
+    ///   `feature_v2` might be in the same logical column.
+    /// - `lancedb:status`: for status options (`production`, `candidate`,
+    ///   `deprecated`, `archived`) to designate the current life cycle state of
+    ///   this column.
     pub async fn update_field_metadata(
         &self,
         updates: &[FieldMetadataUpdate],
@@ -3223,7 +3248,10 @@ impl BaseTable for NativeTable {
         let dataset = self.dataset.get().await?;
         match filter {
             None => Ok(dataset.count_rows(None).await?),
-            Some(Filter::Sql(sql)) => Ok(dataset.count_rows(Some(sql)).await?),
+            Some(Filter::Sql(sql)) => {
+                let sql = crate::expr::canonicalize_sql_predicate(&sql)?;
+                Ok(dataset.count_rows(Some(sql)).await?)
+            }
             Some(Filter::Datafusion(_)) => Err(Error::NotSupported {
                 message: "Datafusion filters are not yet supported".to_string(),
             }),

rust/lancedb/src/table/create_index.rs

@@ -133,7 +133,7 @@ impl NativeTable {
                     ),
                 });
             }
-            (resolved.canonical_path, resolved.terminal_field)
+            (resolved.canonical_path, resolved.field)
         } else {
             Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
         };
@@ -439,8 +439,7 @@ mod tests {
     use arrow_array::record_batch;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Int32Array,
-        LargeBinaryArray, LargeStringArray, ListArray, RecordBatch, StringArray, StructArray,
-        UInt32Array,
+        LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StructArray,
     };
     use arrow_data::ArrayDataBuilder;
     use arrow_schema::{DataType, Field, Schema};
@@ -459,7 +458,6 @@ mod tests {
     use crate::query::{ExecutableQuery, QueryBase};
     use crate::table::optimize::{CompactionOptions, OptimizeAction};
     use lance_index::scalar::FullTextSearchQuery;
-    use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery};
 
     fn create_fixed_size_list<T: Array>(
         values: T,
@@ -601,80 +599,6 @@ mod tests {
         assert!(invalid_granularity.is_err());
     }
 
-    #[tokio::test]
-    async fn test_nested_list_fts_uses_deepest_document_coordinates() {
-        let conn = connect("memory://").execute().await.unwrap();
-        let mut docs = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
-
-        docs.values().values().append_value("alpha");
-        docs.values().values().append_value("beta");
-        docs.values().append(true);
-        docs.values().values().append_value("gamma");
-        docs.values().values().append_value("alpha delta");
-        docs.values().append(true);
-        docs.append(true);
-
-        docs.values().append(true);
-        docs.values().values().append_value("alpha");
-        docs.values().append(true);
-        docs.append(true);
-
-        let batch = RecordBatch::try_from_iter(vec![
-            ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef),
-            ("docs", Arc::new(docs.finish()) as ArrayRef),
-        ])
-        .unwrap();
-        let table = conn.create_table("nested", batch).execute().await.unwrap();
-
-        let job = table
-            .create_index(
-                &["docs"],
-                Index::FTS(
-                    FtsIndexBuilder::default()
-                        .document_granularity(DocumentGranularity::ListElement),
-                ),
-            )
-            .execute_async()
-            .await
-            .unwrap();
-        job.wait().await.unwrap();
-
-        let query = FullTextSearchQuery::new_query(FtsQuery::Match(
-            MatchQuery::new("alpha".to_string())
-                .with_column(Some("docs".to_string()))
-                .with_document_granularity(DocumentGranularity::ListElement),
-        ));
-        let batches = table
-            .query()
-            .full_text_search(query)
-            .limit(10)
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        let mut hits = Vec::new();
-        for batch in batches {
-            let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
-            let coordinates = batch["_doc_index"]
-                .as_any()
-                .downcast_ref::<ListArray>()
-                .unwrap();
-            for row in 0..batch.num_rows() {
-                let coordinate = coordinates.value(row);
-                let coordinate = coordinate.as_any().downcast_ref::<UInt32Array>().unwrap();
-                hits.push((ids.value(row), coordinate.values().to_vec()));
-            }
-        }
-        hits.sort_unstable();
-        assert_eq!(
-            hits,
-            vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]
-        );
-    }
-
     /// Concurrent waiters, and a wait issued after the job settled, all
     /// succeed once the build does.
     #[tokio::test]

rust/lancedb/src/table/datafusion/blob_coerce.rs

@@ -36,6 +36,14 @@ pub(super) fn coerce_blob_expr(
     };
 
     let input_shape = match input_field.data_type() {
+        DataType::Null => {
+            let expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(
+                input_expr,
+                table_field.data_type().clone(),
+                None,
+            ));
+            return Ok((expr, table_field.clone()));
+        }
         DataType::Binary | DataType::LargeBinary | DataType::BinaryView => BlobInputShape::Bytes,
         DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => BlobInputShape::String,
         DataType::Struct(children) => {
@@ -155,7 +163,7 @@ mod tests {
     use crate::blob::blob;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BinaryViewArray, Int32Array, Int64Array, LargeBinaryArray,
-        RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
+        NullArray, RecordBatch, StringArray, StringViewArray, StructArray, UInt8Array, UInt64Array,
     };
     use arrow_schema::Schema;
     use datafusion::prelude::SessionContext;
@@ -279,6 +287,18 @@ mod tests {
         assert_eq!(data.value(0), b"view");
     }
 
+    #[tokio::test]
+    async fn null_column_coerces_to_all_null_blob_struct() {
+        let batch = batch_with_image(
+            Field::new("image", DataType::Null, true),
+            Arc::new(NullArray::new(2)),
+        );
+        let coerced = coerce(batch, &blob_table_schema()).await;
+        let image = image_struct(&coerced);
+        assert!(image.is_null(0));
+        assert!(image.is_null(1));
+    }
+
     #[tokio::test]
     async fn binary_nulls_stay_null_after_coercion() {
         let batch = batch_with_image(

rust/lancedb/src/table/delete.rs

@@ -31,8 +31,9 @@ pub(crate) async fn execute_delete(
     table.dataset.ensure_mutable()?;
     match predicate {
         Predicate::String(s) => {
+            let predicate = crate::expr::canonicalize_sql_predicate(s)?;
             let mut dataset = (*table.dataset.get().await?).clone();
-            let delete_result = dataset.delete(s).boxed().await?;
+            let delete_result = dataset.delete(&predicate).boxed().await?;
             let num_deleted_rows = delete_result.num_deleted_rows;
             let version = dataset.version().version;
             table.dataset.update(dataset);

rust/lancedb/src/table/merge.rs

@@ -220,9 +220,32 @@ impl MergeInsertBuilder {
     ///
     /// Returns version and statistics about the merge operation including the number of rows
     /// inserted, updated, and deleted.
-    pub async fn execute(self, new_data: Box<dyn RecordBatchReader + Send>) -> Result<MergeResult> {
+    pub async fn execute(
+        mut self,
+        new_data: Box<dyn RecordBatchReader + Send>,
+    ) -> Result<MergeResult> {
+        self.canonicalize_filters()?;
         self.table.clone().merge_insert(self, new_data).await
     }
+
+    pub(crate) fn canonicalize_filters(&mut self) -> Result<()> {
+        self.when_matched_update_all_filt =
+            canonicalize_merge_filter(self.when_matched_update_all_filt.take())?;
+        self.when_not_matched_by_source_delete_filt =
+            canonicalize_merge_filter(self.when_not_matched_by_source_delete_filt.take())?;
+        Ok(())
+    }
+}
+
+fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<MergeFilter>> {
+    filter
+        .map(|filter| match filter {
+            MergeFilter::Sql(predicate) => {
+                crate::expr::canonicalize_sql_predicate(&predicate).map(MergeFilter::Sql)
+            }
+            filter @ MergeFilter::Expr(_) => Ok(filter),
+        })
+        .transpose()
 }
 
 /// Internal implementation of the merge insert logic
@@ -230,9 +253,10 @@ impl MergeInsertBuilder {
 /// This logic was moved from NativeTable::merge_insert to keep table.rs clean.
 pub(crate) async fn execute_merge_insert(
     table: &NativeTable,
-    params: MergeInsertBuilder,
+    mut params: MergeInsertBuilder,
     new_data: Box<dyn RecordBatchReader + Send>,
 ) -> Result<MergeResult> {
+    params.canonicalize_filters()?;
     super::computed_columns::ensure_no_function_bindings_for_mutation(
         table.schema().await?.as_ref(),
         "merge_insert",

rust/lancedb/src/table/query.rs

@@ -17,11 +17,11 @@ use arrow::array::{AsArray, FixedSizeListBuilder, Float32Builder};
 use arrow::datatypes::{Float32Type, UInt8Type};
 use arrow_array::Array;
 use arrow_schema::{DataType, Schema};
+use datafusion_common::{Column, DataFusionError, SchemaError};
 use datafusion_physical_plan::ExecutionPlan;
 use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::repartition::RepartitionExec;
 use datafusion_physical_plan::union::UnionExec;
-use futures::future::try_join_all;
 use lance::dataset::mem_wal::DatasetMemWalExt;
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::dataset::scanner::Scanner;
@@ -45,6 +45,22 @@ impl AnyQuery {
             Self::VectorQuery(query) => &query.base,
         }
     }
+
+    fn base_mut(&mut self) -> &mut QueryRequest {
+        match self {
+            Self::Query(query) => query,
+            Self::VectorQuery(query) => &mut query.base,
+        }
+    }
+
+    /// Canonicalize any raw SQL filter immediately before backend dispatch.
+    pub(crate) fn canonicalized(&self) -> Result<Self> {
+        let mut query = self.clone();
+        if let Some(QueryFilter::Sql(predicate)) = &mut query.base_mut().filter {
+            *predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
+        }
+        Ok(query)
+    }
 }
 
 //Decide between namespace or local
@@ -53,15 +69,16 @@ pub async fn execute_query(
     query: &AnyQuery,
     options: QueryExecutionOptions,
 ) -> Result<DatasetRecordBatchStream> {
+    let query = query.canonicalized()?;
     // QueryTable pushdown runs the query server-side, but only on the main
     // branch: the namespace request carries no branch yet, so a branch handle
     // must fall through to local execution.
-    if can_execute_namespace_query(table, query).await?
+    if can_execute_namespace_query(table, &query).await?
         && let Some(ref namespace_client) = table.namespace_client
     {
-        return execute_namespace_query(table, namespace_client.clone(), query, options).await;
+        return execute_namespace_query(table, namespace_client.clone(), &query, options).await;
     }
-    execute_generic_query(table, query, options).await
+    execute_generic_query(table, &query, options).await
 }
 
 async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> Result<bool> {
@@ -136,9 +153,10 @@ pub async fn create_plan(
     query: &AnyQuery,
     options: QueryExecutionOptions,
 ) -> Result<Arc<dyn ExecutionPlan>> {
+    let query = query.canonicalized()?;
     let query = match query {
-        AnyQuery::VectorQuery(query) => query.clone(),
-        AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query.clone()),
+        AnyQuery::VectorQuery(query) => query,
+        AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
     };
     query.base.check_filter()?;
 
@@ -170,26 +188,48 @@ pub async fn create_plan(
     let mut column = query.column.clone();
 
     let mut query_vector = query.query_vector.first().cloned();
+    let mut is_batch_query = false;
     if query.query_vector.len() > 1 {
         if column.is_none() {
             // Infer a vector column with the same dimension of the query vector.
-            let arrow_schema = Schema::from(ds_ref.schema());
+            let arrow_schema = Schema::from(schema);
             column = Some(default_vector_column(
                 &arrow_schema,
                 Some(query.query_vector[0].len() as i32),
             )?);
         }
         let vector_field = schema.field(column.as_ref().unwrap()).unwrap();
-        if let DataType::List(_) = vector_field.data_type() {
-            // Multivector handling: concatenate into FixedSizeList<FixedSizeList<_>>
+        let (_, element_type) =
+            lance::index::vector::utils::get_vector_type(schema, column.as_ref().unwrap())?;
+        let is_binary = matches!(element_type, DataType::UInt8);
+        if matches!(vector_field.data_type(), DataType::List(_))
+            || (query.base.offset.unwrap_or(0) == 0 && !is_binary)
+        {
+            // Lance distinguishes these cases from the vector column type: a
+            // list-like query against a List column is one multivector query,
+            // while the same query against a FixedSizeList column is a batch of
+            // independent queries. The batch path shares a single flat scan and
+            // bounds retained candidate data instead of running one scan per
+            // query vector.
             let vectors = query
                 .query_vector
                 .iter()
                 .map(|arr| arr.as_ref())
                 .collect::<Vec<_>>();
             let dim = vectors[0].len();
+            if let Some((query_index, actual_dim)) = vectors
+                .iter()
+                .enumerate()
+                .find_map(|(index, vector)| (vector.len() != dim).then_some((index, vector.len())))
+            {
+                return Err(Error::InvalidInput {
+                    message: format!(
+                        "query vector at index {query_index} has dimension {actual_dim}, expected {dim}"
+                    ),
+                });
+            }
             let mut fsl_builder = FixedSizeListBuilder::with_capacity(
-                Float32Builder::with_capacity(dim),
+                Float32Builder::with_capacity(dim * vectors.len()),
                 dim as i32,
                 vectors.len(),
             );
@@ -200,8 +240,12 @@ pub async fn create_plan(
                 fsl_builder.append(true);
             }
             query_vector = Some(Arc::new(fsl_builder.finish()));
+            is_batch_query = !matches!(vector_field.data_type(), DataType::List(_));
         } else {
-            // Multiple query vectors: create a plan for each and union them
+            // Lance's batch path has no per-query offset, and its binary path
+            // requires primitive UInt8 queries rather than a fixed-size list.
+            // Keep the prior plan shape for these cases so offsets are applied
+            // per query and binary query vectors retain their primitive shape.
             let query_vecs = query.query_vector.clone();
             let plan_futures = query_vecs
                 .into_iter()
@@ -214,7 +258,7 @@ pub async fn create_plan(
                     }
                 })
                 .collect::<Vec<_>>();
-            let plans = try_join_all(plan_futures).await?;
+            let plans = futures::future::try_join_all(plan_futures).await?;
             return create_multi_vector_plan(plans);
         }
     }
@@ -225,7 +269,7 @@ pub async fn create_plan(
         let column = if let Some(col) = column {
             col
         } else {
-            let arrow_schema = Schema::from(ds_ref.schema());
+            let arrow_schema = Schema::from(schema);
             default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
         };
 
@@ -251,10 +295,14 @@ pub async fn create_plan(
         }
     }
 
-    scanner.limit(
-        query.base.limit.map(|limit| limit as i64),
-        query.base.offset.map(|offset| offset as i64),
-    )?;
+    // For a batch query, `nearest` already applies k to each query vector.
+    // Adding Scanner's global limit would truncate the combined result to k rows.
+    if !is_batch_query {
+        scanner.limit(
+            query.base.limit.map(|limit| limit as i64),
+            query.base.offset.map(|offset| offset as i64),
+        )?;
+    }
 
     if let Some(ef) = query.ef {
         scanner.ef(ef);
@@ -327,7 +375,97 @@ pub async fn create_plan(
         scanner.order_by(Some(order_by.clone()))?;
     }
 
-    Ok(scanner.create_plan().await?)
+    scanner
+        .create_plan()
+        .await
+        .map_err(|error| enrich_lance_field_not_found(error, schema))
+}
+
+/// Replace DataFusion's top-level field candidates with qualified leaf paths.
+///
+/// DataFusion resolves nested fields but its `FieldNotFound` error only lists the
+/// top-level Arrow fields. This makes a missing leaf look unavailable even when it
+/// exists below a struct. Keep every other Lance/DataFusion error unchanged and
+/// enrich only this one schema error at the LanceDB query boundary.
+fn enrich_lance_field_not_found(
+    error: lance::Error,
+    schema: &lance_core::datatypes::Schema,
+) -> Error {
+    let Some(field) = find_missing_field(&error) else {
+        return error.into();
+    };
+    field_not_found_error(field, &Schema::from(schema))
+}
+
+fn field_not_found_diagnostic(
+    error: &(dyn std::error::Error + 'static),
+    schema: &Schema,
+) -> Option<Error> {
+    let field = find_missing_field(error)?;
+    Some(field_not_found_error(field, schema))
+}
+
+fn field_not_found_error(field: &Column, schema: &Schema) -> Error {
+    let valid_fields = leaf_field_paths(schema);
+    let mut message = format!("Schema error: No field named {}", field.quoted_flat_name());
+    if !valid_fields.is_empty() {
+        message.push_str(". Valid fields are ");
+        message.push_str(&valid_fields.join(", "));
+    }
+    message.push('.');
+
+    Error::InvalidInput { message }
+}
+
+fn find_missing_field<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a Column> {
+    if let Some(DataFusionError::SchemaError(schema_error, _)) =
+        error.downcast_ref::<DataFusionError>()
+        && let SchemaError::FieldNotFound { field, .. } = schema_error.as_ref()
+    {
+        return Some(field);
+    }
+
+    error.source().and_then(find_missing_field)
+}
+
+fn leaf_field_paths(schema: &Schema) -> Vec<String> {
+    fn format_segment(segment: &str) -> String {
+        // Quote every segment instead of maintaining a SQL keyword list. Bare
+        // lowercase names such as `true` can be parsed as expressions rather
+        // than identifiers, while backticks preserve all field names in both
+        // local SQL parsers.
+        format!("`{}`", segment.replace('`', "``"))
+    }
+
+    fn visit(fields: &arrow_schema::Fields, path: &mut Vec<String>, paths: &mut Vec<String>) {
+        for field in fields {
+            // Neither local planner can address an empty field-path segment,
+            // even when it is backtick-quoted. Do not advertise leaves beneath
+            // such a segment as valid filter fields.
+            if field.name().is_empty() {
+                continue;
+            }
+            path.push(field.name().clone());
+            match field.data_type() {
+                DataType::Struct(children) if !children.is_empty() => {
+                    visit(children, path, paths);
+                }
+                _ => {
+                    paths.push(
+                        path.iter()
+                            .map(|segment| format_segment(segment))
+                            .collect::<Vec<_>>()
+                            .join("."),
+                    );
+                }
+            }
+            path.pop();
+        }
+    }
+
+    let mut paths = Vec::new();
+    visit(schema.fields(), &mut Vec::new(), &mut paths);
+    paths
 }
 
 //Helper functions below
@@ -687,7 +825,10 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result<DatasetRecordBa
 #[cfg(test)]
 #[allow(deprecated)]
 mod tests {
-    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array};
+    use arrow_array::{
+        ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
+        StructArray,
+    };
     use futures::TryStreamExt;
     use lance_arrow::FixedSizeListArrayExt;
     use std::sync::{
@@ -696,7 +837,7 @@ mod tests {
     };
 
     use super::*;
-    use crate::query::{QueryExecutionOptions, QueryRequest};
+    use crate::query::{ExecutableQuery, QueryBase, QueryExecutionOptions, QueryRequest};
     use crate::table::BaseTable;
 
     fn fixed_size_list_array(values: Vec<f32>, dimension: i32) -> FixedSizeListArray {
@@ -837,7 +978,6 @@ mod tests {
     async fn test_execute_query_local_routing() {
         use crate::connect;
         use crate::table::query::execute_query;
-        use arrow_array::{Int32Array, RecordBatch};
         use arrow_schema::{DataType, Field, Schema};
 
         let conn = connect("memory://").execute().await.unwrap();
@@ -877,6 +1017,164 @@ mod tests {
         assert_eq!(count, 2); // 4 and 5
     }
 
+    #[tokio::test]
+    async fn test_missing_filter_field_lists_nested_fields_in_local_planners() {
+        use crate::connect;
+        use arrow_schema::{DataType, Field, Schema};
+
+        let conn = connect("memory://").execute().await.unwrap();
+        let metadata = Arc::new(StructArray::from(vec![
+            (
+                Arc::new(Field::new("year", DataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![2024])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("genre", DataType::Utf8, false)),
+                Arc::new(StringArray::from(vec!["fiction"])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("Title", DataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![7])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("true", DataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![8])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("", DataType::Int32, false)),
+                Arc::new(Int32Array::from(vec![10])) as ArrayRef,
+            ),
+        ]));
+        let vector = Arc::new(fixed_size_list_array(vec![0.0, 1.0], 2));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("vector", vector.data_type().clone(), false),
+            Field::new("content", DataType::Utf8, false),
+            Field::new("metadata", metadata.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1])),
+                vector,
+                Arc::new(StringArray::from(vec!["example"])),
+                metadata,
+            ],
+        )
+        .unwrap();
+        let table = conn
+            .create_table("nested_error", batch)
+            .execute()
+            .await
+            .unwrap();
+
+        let error = table
+            .query()
+            .only_if("year = 2024")
+            .execute()
+            .await
+            .err()
+            .expect("query should reject the unqualified nested field");
+        let case_sensitive_path = "`metadata`.`Title`";
+        let keyword_path = "`metadata`.`true`";
+        let expected = format!(
+            "No field named year. Valid fields are `id`, `vector`, `content`, `metadata`.`year`, `metadata`.`genre`, {case_sensitive_path}, {keyword_path}."
+        );
+
+        assert!(
+            error.to_string().contains(&expected),
+            "unexpected error: {error}"
+        );
+        for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
+            table
+                .query()
+                .only_if(format!("{path} = {value}"))
+                .execute()
+                .await
+                .expect("the path advertised by the diagnostic should be reusable");
+        }
+
+        table.set_unenforced_primary_key(["id"]).await.unwrap();
+        table
+            .set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded())
+            .await
+            .unwrap();
+        let lsm_error = table
+            .query()
+            .only_if("year = 2024")
+            .execute()
+            .await
+            .err()
+            .expect("LSM query should reject the unqualified nested field");
+
+        assert!(
+            lsm_error.to_string().contains(&expected),
+            "unexpected LSM error: {lsm_error}"
+        );
+        for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
+            table
+                .query()
+                .only_if(format!("{path} = {value}"))
+                .execute()
+                .await
+                .expect("the path advertised by the diagnostic should be reusable in LSM queries");
+        }
+    }
+
+    #[test]
+    fn test_leaf_field_paths_preserve_arbitrary_depth() {
+        use arrow_schema::{DataType, Field, Schema};
+
+        fn nested_field(path: &[&str]) -> Field {
+            let mut segments = path.iter().rev();
+            let mut field = Field::new(
+                *segments.next().expect("path must have a leaf"),
+                DataType::Int32,
+                false,
+            );
+            for segment in segments {
+                field = Field::new(*segment, DataType::Struct(vec![field].into()), false);
+            }
+            field
+        }
+
+        let schema = Schema::new(vec![
+            nested_field(&["a", "b", "c", "d", "e"]),
+            nested_field(&["metadata", "child.with.dot"]),
+            nested_field(&["metadata", "Title"]),
+            nested_field(&["metadata", "123child"]),
+            nested_field(&["metadata", "child`tick"]),
+            nested_field(&["metadata", ""]),
+            nested_field(&["", "child"]),
+        ]);
+
+        assert_eq!(
+            leaf_field_paths(&schema),
+            vec![
+                "`a`.`b`.`c`.`d`.`e`",
+                "`metadata`.`child.with.dot`",
+                "`metadata`.`Title`",
+                "`metadata`.`123child`",
+                "`metadata`.`child``tick`",
+            ]
+        );
+
+        let source = DataFusionError::SchemaError(
+            Box::new(SchemaError::FieldNotFound {
+                field: Box::new(Column::from_name("missing")),
+                valid_fields: Vec::new(),
+            }),
+            Box::new(None),
+        );
+        let error = field_not_found_diagnostic(&source, &schema).unwrap();
+        assert!(
+            error.to_string().contains(
+                "Valid fields are `a`.`b`.`c`.`d`.`e`, `metadata`.`child.with.dot`, `metadata`.`Title`, `metadata`.`123child`, `metadata`.`child``tick`"
+            ),
+            "unexpected error: {error}"
+        );
+    }
+
     #[derive(Debug, Default)]
     struct CountingNamespaceClient {
         query_table_calls: AtomicUsize,
@@ -1088,7 +1386,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_create_plan_multivector_structure() {
+    async fn test_create_plan_batch_vector_uses_shared_scan() {
         use arrow_array::{Float32Array, RecordBatch};
         use arrow_schema::{DataType, Field, Schema};
         use datafusion_physical_plan::display::DisplayableExecutionPlan;
@@ -1115,11 +1413,18 @@ mod tests {
             .unwrap();
         let native_table = table.as_native().unwrap();
 
-        // This triggers the "create_multi_vector_plan" logic branch
+        // A batch of vectors against a fixed-size vector column should use
+        // Lance's native batch KNN path instead of independent scan plans.
         let q1 = Arc::new(Float32Array::from(vec![1.0, 2.0]));
         let q2 = Arc::new(Float32Array::from(vec![3.0, 4.0]));
 
         let req = VectorQueryRequest {
+            base: QueryRequest {
+                filter: Some(QueryFilter::Sql("id >= 0".to_string())),
+                limit: Some(1),
+                select: Select::Columns(vec!["id".to_string()]),
+                ..Default::default()
+            },
             column: Some("vector".to_string()),
             query_vector: vec![q1, q2],
             ..Default::default()
@@ -1136,19 +1441,17 @@ mod tests {
             .indent(true)
             .to_string();
 
-        // We expect a RepartitionExec wrapping a UnionExec
         assert!(
-            display.contains("RepartitionExec"),
-            "Plan should include Repartitioning"
+            display.contains("KNNVectorDistance: queries=2"),
+            "plan should use native batch KNN, got:\n{display}"
         );
         assert!(
-            display.contains("UnionExec"),
-            "Plan should include a Union of multiple searches"
+            !display.contains("UnionExec"),
+            "flat batch KNN should share one scan, got:\n{display}"
         );
-        // We expect the projection to add the 'query_index' column (logic inside multi_vector_plan)
         assert!(
             display.contains("query_index"),
-            "Plan should add query_index column"
+            "plan should add query_index column, got:\n{display}"
         );
     }
 

rust/lancedb/src/table/query/lsm.rs

@@ -27,6 +27,8 @@ use std::sync::Arc;
 
 use arrow_array::Array;
 use arrow_schema::{DataType, Schema as ArrowSchema};
+use datafusion::common::{DataFusionError, ToDFSchema};
+use datafusion::prelude::SessionContext;
 use datafusion_physical_plan::expressions::Column;
 use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
@@ -391,7 +393,21 @@ fn base_scanner(
     }
     if let Some(filter) = &query.base.filter {
         scanner = match filter {
-            QueryFilter::Sql(sql) => scanner.filter(sql)?,
+            QueryFilter::Sql(sql) => {
+                // Parse here instead of inside `LsmScanner::filter` so the typed
+                // DataFusion `FieldNotFound` error is still available for the
+                // same nested-field enrichment used by the ordinary scanner.
+                let schema = ArrowSchema::from(dataset.schema());
+                let df_schema = schema.clone().to_dfschema().map_err(|error| {
+                    enrich_filter_error(error, &schema, "Failed to create DFSchema")
+                })?;
+                let expr = SessionContext::new()
+                    .parse_sql_expr(sql, &df_schema)
+                    .map_err(|error| {
+                        enrich_filter_error(error, &schema, "Failed to parse filter expression")
+                    })?;
+                scanner.filter_expr(expr)
+            }
             QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()),
             QueryFilter::Substrait(_) => {
                 return Err(Error::NotSupported {
@@ -403,6 +419,12 @@ fn base_scanner(
     Ok(scanner)
 }
 
+fn enrich_filter_error(error: DataFusionError, schema: &ArrowSchema, context: &str) -> Error {
+    super::field_not_found_diagnostic(&error, schema).unwrap_or_else(|| Error::InvalidInput {
+        message: format!("{context}: {error}"),
+    })
+}
+
 /// Plain scan: filter / projection / limit over base ∪ SSTables ∪ in-memory.
 /// The plain scan applies limit and offset inside the planner.
 async fn plain_plan(

rust/lancedb/src/table/schema_evolution.rs

@@ -55,7 +55,9 @@ pub struct DropColumnsResult {
 pub struct FieldMetadataUpdate {
     /// Dot-separated path to the field (e.g. `"embedding"` or `"address.zip"`).
     pub path: String,
-    /// Keys to set (`Some`) or delete (`None`).
+    /// Keys to set (`Some`) or delete (`None`). See
+    /// [`Table::update_field_metadata`](crate::Table::update_field_metadata) for
+    /// the conventional `lancedb:*` keys.
     pub metadata: HashMap<String, Option<String>>,
     /// If `true`, replace the field's entire metadata map instead of merging.
     pub replace: bool,

rust/lancedb/src/table/update.rs

@@ -62,22 +62,33 @@ impl UpdateBuilder {
     }
 
     /// Executes the update operation.
-    pub async fn execute(self) -> Result<UpdateResult> {
+    pub async fn execute(mut self) -> Result<UpdateResult> {
         if self.columns.is_empty() {
             Err(Error::InvalidInput {
                 message: "at least one column must be specified in an update operation".to_string(),
             })
         } else {
+            self.canonicalize_filter()?;
             self.parent.clone().update(self).await
         }
     }
+
+    pub(crate) fn canonicalize_filter(&mut self) -> Result<()> {
+        self.filter = self
+            .filter
+            .take()
+            .map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate))
+            .transpose()?;
+        Ok(())
+    }
 }
 
 /// Internal implementation of the update logic
 pub(crate) async fn execute_update(
     table: &NativeTable,
-    update: UpdateBuilder,
+    mut update: UpdateBuilder,
 ) -> Result<UpdateResult> {
+    update.canonicalize_filter()?;
     table.dataset.ensure_mutable()?;
 
     // 1. Snapshot the current dataset

rust/lancedb/src/utils/mod.rs

@@ -227,7 +227,7 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
 
 pub(crate) struct ResolvedFtsField {
     pub canonical_path: String,
-    pub terminal_field: Field,
+    pub field: Field,
     pub list_depth: usize,
 }
 
@@ -309,7 +309,7 @@ pub(crate) fn resolve_lance_fts_field_path(
     );
     Ok(ResolvedFtsField {
         canonical_path,
-        terminal_field: Field::from(terminal),
+        field: Field::from(field),
         list_depth,
     })
 }
@@ -375,7 +375,7 @@ pub(crate) fn resolve_arrow_fts_field_path(
             message: format!("Invalid schema: {}", e),
         })?;
     let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
-    Ok((resolved.canonical_path, resolved.terminal_field))
+    Ok((resolved.canonical_path, resolved.field))
 }
 
 pub fn supported_btree_data_type(dtype: &DataType) -> bool {
@@ -647,9 +647,8 @@ mod tests {
             Field::new("docs", text_list(), true),
         ]);
 
-        let (path, field) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
+        let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
         assert_eq!(path, "docs.content");
-        assert_eq!(field.data_type(), &DataType::Utf8);
 
         let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
         let field_id = lance_schema

rust/lancedb/tests/first_class_function_slice1.rs

@@ -20,6 +20,25 @@ fn job_result(name: &str) -> Value {
     serde_json::from_str::<Value>(&fixture(name)).expect("remote Job fixture")["result"].clone()
 }
 
+fn assert_no_secret_values(value: &Value) {
+    match value {
+        Value::Object(values) => {
+            for (key, value) in values {
+                assert!(
+                    !matches!(
+                        key.as_str(),
+                        "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets"
+                    ),
+                    "client canonical value must not model resolved secret material"
+                );
+                assert_no_secret_values(value);
+            }
+        }
+        Value::Array(values) => values.iter().for_each(assert_no_secret_values),
+        _ => {}
+    }
+}
+
 #[test]
 fn function_version_job_result_matches_shared_canonical_golden() {
     let result = job_result("remote_function_job.json");
@@ -28,6 +47,7 @@ fn function_version_job_result_matches_shared_canonical_golden() {
     assert_eq!(version.name(), "embed");
     assert_eq!(version.version(), "fv_01K3EXACT");
     assert_eq!(version.runtime_digest(), "sha256:runtime");
+    assert_eq!(version.required_secrets(), &["HF_TOKEN"]);
     assert_eq!(
         version.to_canonical_json().expect("canonical JSON"),
         fixture("remote_function_version.canonical.json").trim()
@@ -142,3 +162,21 @@ fn floating_point_application_literals_are_rejected_consistently() {
             .contains("floating-point Function literals")
     );
 }
+
+#[test]
+fn canonical_client_values_contain_secret_names_only() {
+    let result = job_result("remote_function_job.json");
+    let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
+    let canonical: Value = serde_json::from_str(
+        &version
+            .to_canonical_json()
+            .expect("canonical FunctionVersion"),
+    )
+    .expect("canonical JSON");
+
+    assert_eq!(
+        canonical["required_secrets"],
+        serde_json::json!(["HF_TOKEN"])
+    );
+    assert_no_secret_values(&canonical);
+}

rust/lancedb/tests/first_class_function_slice2.rs

@@ -6,6 +6,7 @@ use std::path::PathBuf;
 
 use lancedb::Error;
 use lancedb::function::FunctionRegistrationRequest;
+use serde_json::Value;
 
 fn fixture(name: &str) -> String {
     let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
@@ -14,6 +15,25 @@ fn fixture(name: &str) -> String {
     fs::read_to_string(path).expect("fixture must be readable")
 }
 
+fn assert_no_secret_values(value: &Value) {
+    match value {
+        Value::Object(values) => {
+            for (key, value) in values {
+                assert!(
+                    !matches!(
+                        key.as_str(),
+                        "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets"
+                    ),
+                    "registration requests must not model resolved secret material"
+                );
+                assert_no_secret_values(value);
+            }
+        }
+        Value::Array(values) => values.iter().for_each(assert_no_secret_values),
+        _ => {}
+    }
+}
+
 #[test]
 fn registration_request_matches_shared_canonical_golden() {
     let request = FunctionRegistrationRequest::from_json(&fixture(
@@ -22,10 +42,33 @@ fn registration_request_matches_shared_canonical_golden() {
     .expect("registration request");
     assert_eq!(request.name, "normalize_score");
     assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch");
+    assert_eq!(request.required_secrets, ["API_TOKEN"]);
+    assert!(request.secret_values.is_empty());
     assert_eq!(
         request.to_canonical_json().expect("canonical request"),
         fixture("remote_function_registration_request.canonical.json").trim()
     );
+
+    let value: Value =
+        serde_json::from_str(&request.to_canonical_json().expect("canonical request"))
+            .expect("request JSON");
+    assert_no_secret_values(&value);
+}
+
+#[test]
+fn registration_request_serializes_secret_values_but_redacts_debug_output() {
+    let mut value: Value =
+        serde_json::from_str(&fixture("remote_function_registration_request.json")).unwrap();
+    value["secret_values"] = serde_json::json!({"API_TOKEN": "secret-plaintext"});
+    let request = FunctionRegistrationRequest::from_json(&value.to_string()).unwrap();
+
+    assert_eq!(request.secret_values["API_TOKEN"], "secret-plaintext");
+    let canonical = request.to_canonical_json().unwrap();
+    assert!(canonical.contains("secret-plaintext"));
+    let debug = format!("{request:?}");
+    assert!(debug.contains("API_TOKEN"));
+    assert!(debug.contains("[REDACTED]"));
+    assert!(!debug.contains("secret-plaintext"));
 }
 
 #[tokio::test]

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json

@@ -24,6 +24,7 @@
     },
     "runtime_digest": "sha256:runtime",
     "environment_digest": "sha256:environment",
+    "required_secrets": ["HF_TOKEN"],
     "created_at": "2026-08-21T00:00:00Z"
   },
   "future_job": {"trace_id": "trace-1"}

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.canonical.json

@@ -1 +1 @@
-{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}}
+{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIG5vcm1hbGl6ZV9zY29yZSh2YWx1ZTogZmxvYXQpIC0+IGZsb2F0OgogICAgcmV0dXJuIHZhbHVlIC8gMTAwLjAK","encoding":"base64"},"digest":"sha256:760784bdcef57b802f389b97804cc0b618aae39e86044733451bcd0b13089a7f","entrypoint":"normalize_score","kind":"python_callable"},"name":"normalize_score","required_secrets":["API_TOKEN"],"runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["numpy>=2"]},"kind":"python","python_version":"3.12"},"signature":{"inputs":[{"arrow_type":"float64","name":"value","nullable":false}],"output":{"arrow_type":"float64","kind":"scalar","nullable":false}}}

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_registration_request.json

@@ -39,5 +39,8 @@
     "env": {
       "MODE": "test"
     }
-  }
+  },
+  "required_secrets": [
+    "API_TOKEN"
+  ]
 }

rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json

@@ -1 +1 @@
-{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
+{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","required_secrets":["HF_TOKEN"],"runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}