fix: support nested-list FTS indexing

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.38.0-beta.11"
+current_version = "0.38.0-beta.10"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

Cargo.lock

@@ -5402,7 +5402,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 dependencies = [
  "arrow",
  "async-trait",

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.38.0-beta.11</version>
+    <version>0.38.0-beta.10</version>
 </dependency>
 ```
 

docs/src/js/classes/Table.md

@@ -221,7 +221,7 @@ abstract checkpointLsm(): Promise<void>
 
 Converge this table's LSM write path into its base table.
 
-Freezes once, then triggers compaction and polls until the SSTables that existed
+Seals once, then triggers compaction and polls until the L0 that existed
 at the start is gone. The target set is fixed at the start, so
 generations created *during* the checkpoint are ignored — that is what
 lets it terminate under write load, and what makes it best-effort: it
@@ -289,7 +289,7 @@ It is a no-op when no writers are cached.
 abstract compactLsm(): Promise<void>
 ```
 
-Trigger a background SSTable compaction pass per table shard.
+Trigger a background L0 → base compaction pass per bucket.
 
 Returns once the passes are *dispatched*, not once they finish — watch
 [Table#getLsmStats](Table.md#getlsmstats) for progress, or use
@@ -505,7 +505,7 @@ Drop an index from the table.
 abstract flushLsm(): Promise<void>
 ```
 
-Freeze every table shard's active memtable into a new SSTable.
+Seal every bucket's active memtable into a new L0 generation.
 
 Returns once the seal is committed. Sealing an empty memtable is a no-op,
 so this is safe to call repeatedly.
@@ -519,10 +519,10 @@ so this is safe to call repeatedly.
 ### getLsmStats()
 
 ```ts
-abstract getLsmStats(includeSstableRows?): Promise<undefined | LsmStats>
+abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
 ```
 
-Read live per-table-shard LSM state.
+Read live per-bucket LSM state.
 
 Answers "how far behind is my fresh tier", "which bucket is hot", and
 "why is my fresh-tier vector search brute-force". Mutates no table state.
@@ -531,8 +531,8 @@ Resolves to `undefined` only when the LSM write path is not enabled.
 
 #### Parameters
 
-* **includeSstableRows?**: `boolean`
-    Also count rows per SSTable.
+* **includeGenerationRows?**: `boolean`
+    Also count rows per L0 generation.
     Off by default because each count opens an uncached Lance dataset.
 
 #### Returns
@@ -1292,18 +1292,6 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
 
 Update per-field (column) metadata.
 
-The following keys are treated specially, by convention, and should be
-used when appropriate:
-
-- `lancedb:description`: for a human-readable description of a field.
-- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
-  names the tag category; e.g. `lancedb:tag:model: "clip"`.
-- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
-  `feature_v2` might be in the same logical column.
-- `lancedb:status`: for status options (`production`, `candidate`,
-  `deprecated`, `archived`) to designate the current life cycle state of
-  this column.
-
 #### Parameters
 
 * **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]

docs/src/js/globals.md

@@ -60,6 +60,7 @@
 - [BranchDiff](interfaces/BranchDiff.md)
 - [BranchIndexSummary](interfaces/BranchIndexSummary.md)
 - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
+- [BucketStats](interfaces/BucketStats.md)
 - [CherryPickError](interfaces/CherryPickError.md)
 - [CherryPickPreview](interfaces/CherryPickPreview.md)
 - [CherryPickResult](interfaces/CherryPickResult.md)
@@ -86,6 +87,7 @@
 - [FtsToken](interfaces/FtsToken.md)
 - [FullTextQuery](interfaces/FullTextQuery.md)
 - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
+- [GenerationStats](interfaces/GenerationStats.md)
 - [HnswPqOptions](interfaces/HnswPqOptions.md)
 - [HnswSqOptions](interfaces/HnswSqOptions.md)
 - [IndexConfig](interfaces/IndexConfig.md)
@@ -124,9 +126,7 @@
 - [SplitHashOptions](interfaces/SplitHashOptions.md)
 - [SplitRandomOptions](interfaces/SplitRandomOptions.md)
 - [SplitSequentialOptions](interfaces/SplitSequentialOptions.md)
-- [SsTableStats](interfaces/SsTableStats.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
-- [TableShardStats](interfaces/TableShardStats.md)
 - [TableStatistics](interfaces/TableStatistics.md)
 - [TimeoutConfig](interfaces/TimeoutConfig.md)
 - [TlsConfig](interfaces/TlsConfig.md)

docs/src/js/interfaces/BucketStats.md

@@ -2,12 +2,12 @@
 
 ***
 
-[@lancedb/lancedb](../globals.md) / TableShardStats
+[@lancedb/lancedb](../globals.md) / BucketStats
 
-# Interface: TableShardStats
+# Interface: BucketStats
 
-Live state of one table shard. A table is N table shards on one node; flattening to a
-single number hides the one hot table shard that is usually why someone opened
+Live state of one bucket. A table is N buckets on one node; flattening to a
+single number hides the one hot bucket that is usually why someone opened
 this endpoint.
 
 ## Properties
@@ -18,7 +18,7 @@ this endpoint.
 compacting: boolean;
 ```
 
-Whether a pass owns this table shard's compaction latch right now. Says *a*
+Whether a pass owns this bucket's compaction latch right now. Says *a*
 driver is running, not *whose*, and the latch is held from dispatch —
 including while the pass queues for a pod-wide compactor permit. Read it
 as "do not pile on", never as "mine is progressing".
@@ -35,13 +35,13 @@ The generation the active memtable will become.
 
 ***
 
-### sstables
+### generations
 
 ```ts
-sstables: SsTableStats[];
+generations: GenerationStats[];
 ```
 
-SSTables not yet merged into the base table.
+Flushed L0 generations not yet merged into the base table.
 
 ***
 
@@ -61,7 +61,7 @@ Version of the shard manifest these numbers were read from.
 optional memtables: MemtableStats[];
 ```
 
-Oldest first, active last. Absent for a `"Sealed"` table shard, whose
+Oldest first, active last. Absent for a `"Sealed"` bucket, whose
 in-memory state is torn down.
 
 ***
@@ -82,7 +82,7 @@ WAL position replay resumes from.
 shardId: string;
 ```
 
-The shard this table shard writes.
+The shard this bucket writes.
 
 ***
 

docs/src/js/interfaces/FieldMetadataUpdate.md

@@ -17,8 +17,7 @@ metadata: Record<string, null | string>;
 ```
 
 Metadata key/value pairs. Merged into the field's existing metadata by
-default; a value of `null` deletes that key. See
-[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
+default; a value of `null` deletes that key.
 
 ***
 

docs/src/js/interfaces/GenerationStats.md

@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / GenerationStats
+
+# Interface: GenerationStats
+
+One flushed L0 generation.
+
+## Properties
+
+### bytes
+
+```ts
+bytes: number;
+```
+
+On-disk size of the generation.
+
+***
+
+### generation
+
+```ts
+generation: number;
+```
+
+The generation number. Increases as memtables are sealed into L0.
+
+***
+
+### rows?
+
+```ts
+optional rows: number;
+```
+
+Present only when `includeGenerationRows` was requested. Off by default
+because each count opens an uncached Lance dataset.

docs/src/js/interfaces/LsmStats.md

@@ -6,17 +6,17 @@
 
 # Interface: LsmStats
 
-Live per-table-shard LSM state, as returned by `Table#getLsmStats`.
+Live per-bucket LSM state, as returned by `Table#getLsmStats`.
 
-Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
+Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
 the caller's to compute.
 
 ## Properties
 
-### tableShards
+### buckets
 
 ```ts
-tableShards: TableShardStats[];
+buckets: BucketStats[];
 ```
 
-One entry per table shard backing this table.
+One entry per bucket backing this table.

docs/src/js/interfaces/SsTableStats.md

@@ -1,40 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / SsTableStats
-
-# Interface: SsTableStats
-
-One SSTable.
-
-## Properties
-
-### bytes
-
-```ts
-bytes: number;
-```
-
-On-disk size of the SSTable.
-
-***
-
-### generation
-
-```ts
-generation: number;
-```
-
-The generation number. Increases as memtables are frozen into SSTables.
-
-***
-
-### rows?
-
-```ts
-optional rows: number;
-```
-
-Present only when `includeSstableRows` was requested. Off by default
-because each count opens an uncached Lance dataset.

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.38.0-beta.11</version>
+      <version>0.38.0-beta.10</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/lancedb-core/src/main/java/com/lancedb/BucketStats.java

@@ -22,11 +22,11 @@ import java.util.Optional;
 import java.util.OptionalLong;
 
 /**
- * Live state of one tableShard. A table is N tableShards on one node; flattening to a single number hides
- * the one hot tableShard that is usually why someone opened this endpoint.
+ * Live state of one bucket. A table is N buckets on one node; flattening to a single number hides
+ * the one hot bucket that is usually why someone opened this endpoint.
  */
-public class TableShardStats {
-  private static final String CONTEXT = "tableShard stats";
+public class BucketStats {
+  private static final String CONTEXT = "bucket stats";
 
   private final String shardId;
   private final String status;
@@ -35,11 +35,11 @@ public class TableShardStats {
   private final long currentGeneration;
   private final long replayAfterWalEntryPosition;
   private final long walEntryPositionLastSeen;
-  private final List<SsTableStats> sstables;
+  private final List<GenerationStats> generations;
   private final boolean compacting;
   private final List<MemtableStats> memtables;
 
-  TableShardStats(
+  BucketStats(
       String shardId,
       String status,
       long writerEpoch,
@@ -47,7 +47,7 @@ public class TableShardStats {
       long currentGeneration,
       long replayAfterWalEntryPosition,
       long walEntryPositionLastSeen,
-      List<SsTableStats> sstables,
+      List<GenerationStats> generations,
       boolean compacting,
       List<MemtableStats> memtables) {
     this.shardId = shardId;
@@ -57,12 +57,12 @@ public class TableShardStats {
     this.currentGeneration = currentGeneration;
     this.replayAfterWalEntryPosition = replayAfterWalEntryPosition;
     this.walEntryPositionLastSeen = walEntryPositionLastSeen;
-    this.sstables = Collections.unmodifiableList(sstables);
+    this.generations = Collections.unmodifiableList(generations);
     this.compacting = compacting;
     this.memtables = memtables == null ? null : Collections.unmodifiableList(memtables);
   }
 
-  /** The shard this tableShard writes. */
+  /** The shard this bucket writes. */
   public String shardId() {
     return shardId;
   }
@@ -100,13 +100,13 @@ public class TableShardStats {
     return walEntryPositionLastSeen;
   }
 
-  /** SSTables not yet merged into the base table. */
-  public List<SsTableStats> sstables() {
-    return sstables;
+  /** Flushed L0 generations not yet merged into the base table. */
+  public List<GenerationStats> generations() {
+    return generations;
   }
 
   /**
-   * Whether a pass owns this tableShard's compaction latch right now. Says <em>a</em> driver is
+   * Whether a pass owns this bucket's compaction latch right now. Says <em>a</em> driver is
    * running, not <em>whose</em>, and the latch is held from dispatch — including while the pass
    * queues for a pod-wide compactor permit. Read it as "do not pile on", never as "mine is
    * progressing".
@@ -115,15 +115,15 @@ public class TableShardStats {
     return compacting;
   }
 
-  /** Oldest first, active last. Empty for a {@code "Sealed"} tableShard, whose state is torn down. */
+  /** Oldest first, active last. Empty for a {@code "Sealed"} bucket, whose state is torn down. */
   public Optional<List<MemtableStats>> memtables() {
     return Optional.ofNullable(memtables);
   }
 
-  /** The newest SSTable generation, or empty when the tier is empty. */
-  OptionalLong newestSstableGeneration() {
+  /** The newest flushed generation, or empty when L0 is empty. */
+  OptionalLong newestGeneration() {
     OptionalLong newest = OptionalLong.empty();
-    for (SsTableStats generation : sstables) {
+    for (GenerationStats generation : generations) {
       if (!newest.isPresent() || generation.generation() > newest.getAsLong()) {
         newest = OptionalLong.of(generation.generation());
       }
@@ -132,15 +132,15 @@ public class TableShardStats {
   }
 
   /**
-   * How many SSTables at or below {@code target} are still uncompacted.
+   * How many generations at or below {@code target} are still in L0.
    *
    * <p>A count, not a boolean: one pass drains a bounded prefix rather than the whole target set,
    * so a boolean would read as "no progress" for every pass but the last. Compaction drains
    * oldest-first, so this decreases monotonically.
    */
-  long outstandingSstables(long target) {
+  long outstandingGenerations(long target) {
     long count = 0;
-    for (SsTableStats generation : sstables) {
+    for (GenerationStats generation : generations) {
       if (generation.generation() <= target) {
         count++;
       }
@@ -148,11 +148,11 @@ public class TableShardStats {
     return count;
   }
 
-  static TableShardStats fromJson(JsonNode node) {
+  static BucketStats fromJson(JsonNode node) {
     JsonFields.requiredObject(node, CONTEXT);
-    List<SsTableStats> sstables = new ArrayList<SsTableStats>();
-    for (JsonNode generation : JsonFields.requiredArray(node, "sstables", CONTEXT)) {
-      sstables.add(SsTableStats.fromJson(generation));
+    List<GenerationStats> generations = new ArrayList<GenerationStats>();
+    for (JsonNode generation : JsonFields.requiredArray(node, "generations", CONTEXT)) {
+      generations.add(GenerationStats.fromJson(generation));
     }
 
     JsonNode memtablesNode = JsonFields.optionalArray(node, "memtables", CONTEXT);
@@ -164,7 +164,7 @@ public class TableShardStats {
       }
     }
 
-    return new TableShardStats(
+    return new BucketStats(
         JsonFields.requiredText(node, "shard_id", CONTEXT),
         JsonFields.requiredText(node, "status", CONTEXT),
         JsonFields.requiredLong(node, "writer_epoch", CONTEXT),
@@ -172,21 +172,21 @@ public class TableShardStats {
         JsonFields.requiredLong(node, "current_generation", CONTEXT),
         JsonFields.requiredLong(node, "replay_after_wal_entry_position", CONTEXT),
         JsonFields.requiredLong(node, "wal_entry_position_last_seen", CONTEXT),
-        sstables,
+        generations,
         JsonFields.requiredBoolean(node, "compacting", CONTEXT),
         memtables);
   }
 
   @Override
   public String toString() {
-    return "TableShardStats{shardId="
+    return "BucketStats{shardId="
         + shardId
         + ", status="
         + status
         + ", currentGeneration="
         + currentGeneration
-        + ", sstables="
-        + sstables
+        + ", generations="
+        + generations
         + ", compacting="
         + compacting
         + "}";

java/lancedb-core/src/main/java/com/lancedb/GenerationStats.java

@@ -17,21 +17,21 @@ import com.fasterxml.jackson.databind.JsonNode;
 
 import java.util.OptionalLong;
 
-/** One SSTable. */
-public class SsTableStats {
+/** One flushed L0 generation. */
+public class GenerationStats {
   private static final String CONTEXT = "generation stats";
 
   private final long generation;
   private final long bytes;
   private final Long rows;
 
-  SsTableStats(long generation, long bytes, Long rows) {
+  GenerationStats(long generation, long bytes, Long rows) {
     this.generation = generation;
     this.bytes = bytes;
     this.rows = rows;
   }
 
-  /** The generation number. Increases as memtables are frozen into SSTables. */
+  /** The generation number. Increases as memtables are sealed into L0. */
   public long generation() {
     return generation;
   }
@@ -42,16 +42,16 @@ public class SsTableStats {
   }
 
   /**
-   * Rows in this generation, present only when {@code includeSstableRows} was requested. Off by
+   * Rows in this generation, present only when {@code includeGenerationRows} was requested. Off by
    * default because each count opens an uncached Lance dataset.
    */
   public OptionalLong rows() {
     return rows == null ? OptionalLong.empty() : OptionalLong.of(rows);
   }
 
-  static SsTableStats fromJson(JsonNode node) {
+  static GenerationStats fromJson(JsonNode node) {
     JsonFields.requiredObject(node, CONTEXT);
-    return new SsTableStats(
+    return new GenerationStats(
         JsonFields.requiredLong(node, "generation", CONTEXT),
         JsonFields.requiredLong(node, "bytes", CONTEXT),
         JsonFields.optionalLong(node, "rows", CONTEXT));
@@ -59,6 +59,6 @@ public class SsTableStats {
 
   @Override
   public String toString() {
-    return "SsTableStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}";
+    return "GenerationStats{generation=" + generation + ", bytes=" + bytes + ", rows=" + rows + "}";
   }
 }

java/lancedb-core/src/main/java/com/lancedb/LanceDbTableLsm.java

@@ -26,7 +26,7 @@ import java.util.OptionalLong;
  *
  * <p>Installing an {@link LsmWriteSpec} routes {@code mergeInsert} upserts through Lance's MemWAL —
  * an LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable,
- * freeze into SSTables, and are merged into the base table by compaction.
+ * seal into L0 generations, and are merged into the base table by compaction.
  *
  * <p>These routes are not part of the Lance Namespace specification, so they are issued directly
  * rather than through {@link org.lance.namespace.LanceNamespace}.
@@ -38,7 +38,7 @@ import java.util.OptionalLong;
  *     .buildRestClient();
  *
  * LanceDbTableLsm lsm = new LanceDbTableLsm(client, "my_table");
- * lsm.setLsmWriteSpec(LsmWriteSpec.tableShard("id", 16));
+ * lsm.setLsmWriteSpec(LsmWriteSpec.bucket("id", 16));
  * // ... merge_insert traffic ...
  * lsm.checkpointLsm();
  * }</pre>
@@ -94,7 +94,7 @@ public class LanceDbTableLsm {
    * Install an {@link LsmWriteSpec} on this table, selecting the MemWAL LSM write path for future
    * {@code mergeInsert} calls.
    *
-   * <p>All variants require the table to have an unenforced primary key; tableShard sharding
+   * <p>All variants require the table to have an unenforced primary key; bucket sharding
    * additionally requires it to be the single column being bucketed.
    */
   public void setLsmWriteSpec(LsmWriteSpec spec) {
@@ -130,7 +130,7 @@ public class LanceDbTableLsm {
   }
 
   /**
-   * Freeze every table shard's active memtable into a new SSTable.
+   * Seal every bucket's active memtable into a new L0 generation.
    *
    * <p>Returns once the seal is committed. Sealing an empty memtable is a no-op, so this is safe to
    * call repeatedly.
@@ -140,7 +140,7 @@ public class LanceDbTableLsm {
   }
 
   /**
-   * Trigger a background SSTable compaction pass per table shard.
+   * Trigger a background L0 → base compaction pass per bucket.
    *
    * <p>Returns once the passes are <em>dispatched</em>, not once they finish — watch {@link
    * #getLsmStats}, or use {@link #checkpointLsm} to wait for convergence.
@@ -150,9 +150,9 @@ public class LanceDbTableLsm {
   }
 
   /**
-   * Read live per-tableShard LSM state.
+   * Read live per-bucket LSM state.
    *
-   * <p>Answers "how far behind is my fresh tier", "which tableShard is hot", and "why is my fresh-tier
+   * <p>Answers "how far behind is my fresh tier", "which bucket is hot", and "why is my fresh-tier
    * vector search brute-force". Mutates no table state.
    *
    * <p>Empty only when the LSM write path is not enabled — that is, when the server sends an absent
@@ -160,13 +160,13 @@ public class LanceDbTableLsm {
    * one throws rather than decoding to something empty, because {@link #checkpointLsm} reads
    * convergence out of these numbers and cannot tell a defaulted array from a drained one.
    *
-   * @param includeSstableRows Also count rows per SSTable. Off by default because each
+   * @param includeGenerationRows Also count rows per L0 generation. Off by default because each
    *     count opens an uncached Lance dataset.
    * @throws IllegalStateException if the response is absent or does not decode.
    */
-  public Optional<LsmStats> getLsmStats(boolean includeSstableRows) {
+  public Optional<LsmStats> getLsmStats(boolean includeGenerationRows) {
     Map<String, Object> body = new LinkedHashMap<String, Object>();
-    body.put("include_sstable_rows", includeSstableRows);
+    body.put("include_generation_rows", includeGenerationRows);
     JsonNode response = client.post(route("get_lsm_stats"), body);
     if (response == null) {
       throw new IllegalStateException("get_lsm_stats returned an empty response body");
@@ -186,8 +186,8 @@ public class LanceDbTableLsm {
   /**
    * Converge this table's LSM write path into its base table.
    *
-   * <p>Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction and
-   * polls until those SSTables are gone. The target set is fixed at the start, so sstables created
+   * <p>Seals once, fixes a target watermark from the resulting L0, then triggers compaction and
+   * polls until that L0 is gone. The target set is fixed at the start, so generations created
    * <em>during</em> the checkpoint are ignored — that is what lets it terminate under write load,
    * and what makes it best-effort: it converges the fresh tier as of some instant. Idempotent,
    * abandonable at any point, safe on a cadence.
@@ -204,7 +204,7 @@ public class LanceDbTableLsm {
     for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
       // The seal turns everything written before this call into a generation, so the
       // watermark has to be read after it. Idempotent: sealing an empty memtable is a
-      // no-op, so a re-issue does not churn empty sstables.
+      // no-op, so a re-issue does not churn empty generations.
       if (issueVoid(this::flushLsm)) {
         backoff(reissue);
         continue;
@@ -220,7 +220,7 @@ public class LanceDbTableLsm {
         return;
       }
 
-      Map<String, Long> targets = newestSstableGenerations(stats.value.get());
+      Map<String, Long> targets = newestGenerations(stats.value.get());
       if (targets.isEmpty()) {
         return;
       }
@@ -236,7 +236,7 @@ public class LanceDbTableLsm {
   }
 
   /**
-   * Trigger and poll until no tableShard holds a generation at or below its target.
+   * Trigger and poll until no bucket holds a generation at or below its target.
    *
    * @return true when the drain finished, false when the table needs re-claiming from flush.
    */
@@ -250,21 +250,21 @@ public class LanceDbTableLsm {
         return true;
       }
 
-      // `compacting` is the tableShard's compaction latch, held from dispatch until the pass
+      // `compacting` is the bucket's compaction latch, held from dispatch until the pass
       // ends — including while it waits on a pod-wide permit. So it answers one question
       // only: do not pile on. Buckets with nothing outstanding are skipped, not counted
       // as idle.
       long outstanding = 0;
       boolean allCompacting = true;
-      for (TableShardStats tableShard : stats.value.get().tableShards()) {
-        Long target = targets.get(tableShard.shardId());
+      for (BucketStats bucket : stats.value.get().buckets()) {
+        Long target = targets.get(bucket.shardId());
         if (target == null) {
           continue;
         }
-        long remaining = tableShard.outstandingSstables(target);
+        long remaining = bucket.outstandingGenerations(target);
         if (remaining > 0) {
           outstanding += remaining;
-          allCompacting &= tableShard.compacting();
+          allCompacting &= bucket.compacting();
         }
       }
       if (outstanding == 0) {
@@ -281,7 +281,7 @@ public class LanceDbTableLsm {
           if (!isRetryable(e)) {
             throw e;
           }
-          // A 429 here means the server could latch no tableShard at all, which the poll
+          // A 429 here means the server could latch no bucket at all, which the poll
           // above already handles. Not retried in place: the latch it would contend for
           // is the one doing the work, so fall through and re-read — POLL_INTERVAL_MS is
           // the backoff.
@@ -291,13 +291,13 @@ public class LanceDbTableLsm {
     }
   }
 
-  /** The newest generation held by each tableShard, skipping tableShards holding none. */
-  private static Map<String, Long> newestSstableGenerations(LsmStats stats) {
+  /** The newest generation held by each bucket, skipping buckets holding none. */
+  private static Map<String, Long> newestGenerations(LsmStats stats) {
     Map<String, Long> targets = new HashMap<String, Long>();
-    for (TableShardStats tableShard : stats.tableShards()) {
-      OptionalLong newest = tableShard.newestSstableGeneration();
+    for (BucketStats bucket : stats.buckets()) {
+      OptionalLong newest = bucket.newestGeneration();
       if (newest.isPresent()) {
-        targets.put(tableShard.shardId(), newest.getAsLong());
+        targets.put(bucket.shardId(), newest.getAsLong());
       }
     }
     return targets;

java/lancedb-core/src/main/java/com/lancedb/LsmStats.java

@@ -20,37 +20,37 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * Live per-tableShard LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
+ * Live per-bucket LSM state, as returned by {@link LanceDbTableLsm#getLsmStats()}.
  *
- * <p>Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to
+ * <p>Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are the caller's to
  * compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
  * because a stats object of zeros would read as measurements.
  */
 public class LsmStats {
   private static final String CONTEXT = "lsm stats";
 
-  private final List<TableShardStats> tableShards;
+  private final List<BucketStats> buckets;
 
-  LsmStats(List<TableShardStats> tableShards) {
-    this.tableShards = Collections.unmodifiableList(tableShards);
+  LsmStats(List<BucketStats> buckets) {
+    this.buckets = Collections.unmodifiableList(buckets);
   }
 
-  /** One entry per tableShard. */
-  public List<TableShardStats> tableShards() {
-    return tableShards;
+  /** One entry per bucket. */
+  public List<BucketStats> buckets() {
+    return buckets;
   }
 
   static LsmStats fromJson(JsonNode node) {
     JsonFields.requiredObject(node, CONTEXT);
-    List<TableShardStats> tableShards = new ArrayList<TableShardStats>();
-    for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) {
-      tableShards.add(TableShardStats.fromJson(tableShard));
+    List<BucketStats> buckets = new ArrayList<BucketStats>();
+    for (JsonNode bucket : JsonFields.requiredArray(node, "buckets", CONTEXT)) {
+      buckets.add(BucketStats.fromJson(bucket));
     }
-    return new LsmStats(tableShards);
+    return new LsmStats(buckets);
   }
 
   @Override
   public String toString() {
-    return "LsmStats{tableShards=" + tableShards + "}";
+    return "LsmStats{buckets=" + buckets + "}";
   }
 }

java/lancedb-core/src/test/java/com/lancedb/LanceDbTableLsmTest.java

@@ -132,10 +132,10 @@ public class LanceDbTableLsmTest {
     enqueue("set_lsm_write_spec", 200, "");
 
     lsm.setLsmWriteSpec(
-        LsmWriteSpec.tableShard("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
+        LsmWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
 
     JsonNode body = MAPPER.readTree(requestBodies.get(0));
-    assertEquals("tableShard", body.get("sharding").get("mode").asText());
+    assertEquals("bucket", body.get("sharding").get("mode").asText());
     assertEquals("id", body.get("sharding").get("column").asText());
     assertEquals(16, body.get("sharding").get("num_buckets").asInt());
     assertEquals(1, body.get("maintained_indexes").size());
@@ -201,7 +201,7 @@ public class LanceDbTableLsmTest {
     enqueue(
         "get_lsm_write_spec",
         200,
-        "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"tableShard\",\"column\":\"id\","
+        "{\"lsm_write_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\","
             + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"],"
             + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}");
 
@@ -228,14 +228,14 @@ public class LanceDbTableLsmTest {
 
   @Test
   public void testGetLsmStats() throws Exception {
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
 
     Optional<LsmStats> got = lsm.getLsmStats(true);
 
     assertEquals("/v1/table/my_table/get_lsm_stats/", requestPaths.get(0));
-    assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
+    assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
     assertTrue(got.isPresent());
-    TableShardStats decoded = got.get().tableShards().get(0);
+    BucketStats decoded = got.get().buckets().get(0);
     assertEquals("shard-0", decoded.shardId());
     assertEquals("Active", decoded.status());
     assertEquals(1, decoded.writerEpoch());
@@ -243,8 +243,8 @@ public class LanceDbTableLsmTest {
     assertEquals(9, decoded.currentGeneration());
     assertFalse(decoded.compacting());
     assertEquals(Arrays.asList(7L, 8L), generationNumbers(decoded));
-    assertEquals(1024, decoded.sstables().get(0).bytes());
-    assertFalse(decoded.sstables().get(0).rows().isPresent(), "rows absent unless requested");
+    assertEquals(1024, decoded.generations().get(0).bytes());
+    assertFalse(decoded.generations().get(0).rows().isPresent(), "rows absent unless requested");
     assertFalse(decoded.memtables().isPresent(), "absent memtables stay absent");
   }
 
@@ -254,19 +254,19 @@ public class LanceDbTableLsmTest {
     enqueue(
         "get_lsm_stats",
         200,
-        "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
             + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
             + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11,"
-            + "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
+            + "\"generations\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
             + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5,"
             + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}");
 
-    TableShardStats decoded = lsm.getLsmStats(true).get().tableShards().get(0);
+    BucketStats decoded = lsm.getLsmStats(true).get().buckets().get(0);
 
     assertEquals(3, decoded.replayAfterWalEntryPosition());
     assertEquals(11, decoded.walEntryPositionLastSeen());
     assertTrue(decoded.compacting());
-    assertEquals(42, decoded.sstables().get(0).rows().getAsLong());
+    assertEquals(42, decoded.generations().get(0).rows().getAsLong());
     assertTrue(decoded.memtables().isPresent());
     MemtableStats memtable = decoded.memtables().get().get(0);
     assertEquals(8, memtable.generation());
@@ -289,7 +289,7 @@ public class LanceDbTableLsmTest {
 
     lsm.getLsmStats();
 
-    assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
+    assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_generation_rows").asBoolean());
   }
 
   // ===========================================================================
@@ -334,8 +334,8 @@ public class LanceDbTableLsmTest {
   @Test
   public void testCheckpointReturnsWhenNoGenerationsOutstanding() {
     enqueue("flush_lsm", 200, "");
-    // A table shard with no SSTables yields no target, so the drain never starts.
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
+    // A bucket with no L0 generations yields no target, so the drain never starts.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
 
     lsm.checkpointLsm();
 
@@ -345,12 +345,12 @@ public class LanceDbTableLsmTest {
   @Test
   public void testCheckpointConvergesOnceTargetGenerationsAreGone() {
     enqueue("flush_lsm", 200, "");
-    // Watermark read: shard-0 holds sstables 7 and 8, so target = 8.
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
+    // Watermark read: shard-0 holds generations 7 and 8, so target = 8.
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
     // First drain poll: both still outstanding, nothing compacting -> dispatch a pass.
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 7L, 8L)));
     // Second drain poll: drained past the target -> done.
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 9L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 9L)));
     enqueue("compact_lsm", 200, "");
 
     lsm.checkpointLsm();
@@ -362,14 +362,14 @@ public class LanceDbTableLsmTest {
   @Test
   public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() {
     enqueue("flush_lsm", 200, "");
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
     // Still compacting on the first poll, so no pass is dispatched; then it drains.
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", true, 4L)));
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false, 5L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", true, 4L)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false, 5L)));
 
     lsm.checkpointLsm();
 
-    assertEquals(0, countCalls("compact_lsm"), "a latched tableShard is left alone");
+    assertEquals(0, countCalls("compact_lsm"), "a latched bucket is left alone");
   }
 
   @Test
@@ -378,7 +378,7 @@ public class LanceDbTableLsmTest {
     // from flush rather than retrying the read in place.
     enqueue("flush_lsm", 200, "");
     enqueue("get_lsm_stats", 421, "no claim");
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
 
     lsm.checkpointLsm();
 
@@ -389,7 +389,7 @@ public class LanceDbTableLsmTest {
   public void testCheckpointRetriesRetryableStatusInPlace() {
     enqueue("flush_lsm", 429, "latch held");
     enqueue("flush_lsm", 200, "");
-    enqueue("get_lsm_stats", 200, stats(tableShard("shard-0", false)));
+    enqueue("get_lsm_stats", 200, stats(bucket("shard-0", false)));
 
     lsm.checkpointLsm();
 
@@ -421,27 +421,27 @@ public class LanceDbTableLsmTest {
 
   /**
    * A stats payload that does not decode must fail closed. Every one of these bodies used to be
-   * read as "no tableShards", which is indistinguishable from a drained table, so {@code checkpointLsm}
+   * read as "no buckets", which is indistinguishable from a drained table, so {@code checkpointLsm}
    * reported convergence for a checkpoint that never ran.
    */
   @Test
   public void testCheckpointRejectsMalformedStats() {
     Map<String, String> malformed = new LinkedHashMap<String, String>();
     malformed.put("no response body at all", "");
-    malformed.put("stats object with no tableShards", "{\"lsm_stats\":{}}");
-    malformed.put("tableShard missing its required fields", "{\"lsm_stats\":{\"tableShards\":[{}]}}");
+    malformed.put("stats object with no buckets", "{\"lsm_stats\":{}}");
+    malformed.put("bucket missing its required fields", "{\"lsm_stats\":{\"buckets\":[{}]}}");
     malformed.put(
-        "tableShard missing sstables",
-        "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+        "bucket missing generations",
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
             + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
             + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
             + "\"compacting\":false}]}}");
     malformed.put(
         "generation with a non-numeric generation number",
-        "{\"lsm_stats\":{\"tableShards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+        "{\"lsm_stats\":{\"buckets\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
             + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
             + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
-            + "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}],"
+            + "\"generations\":[{\"generation\":\"7\",\"bytes\":1024}],"
             + "\"compacting\":false}]}}");
 
     for (Map.Entry<String, String> each : malformed.entrySet()) {
@@ -492,22 +492,22 @@ public class LanceDbTableLsmTest {
   // harness
   // ===========================================================================
 
-  private static List<Long> generationNumbers(TableShardStats tableShard) {
+  private static List<Long> generationNumbers(BucketStats bucket) {
     List<Long> numbers = new ArrayList<Long>();
-    for (SsTableStats generation : tableShard.sstables()) {
+    for (GenerationStats generation : bucket.generations()) {
       numbers.add(generation.generation());
     }
     return numbers;
   }
 
-  /** Build an {@code lsm_stats} response body from tableShard fragments. */
-  private static String stats(String... tableShards) {
-    return "{\"lsm_stats\":{\"tableShards\":[" + String.join(",", tableShards) + "]}}";
+  /** Build an {@code lsm_stats} response body from bucket fragments. */
+  private static String stats(String... buckets) {
+    return "{\"lsm_stats\":{\"buckets\":[" + String.join(",", buckets) + "]}}";
   }
 
-  private static String tableShard(String shardId, boolean compacting, Long... sstables) {
+  private static String bucket(String shardId, boolean compacting, Long... generations) {
     StringBuilder gens = new StringBuilder();
-    for (Long generation : sstables) {
+    for (Long generation : generations) {
       if (gens.length() > 0) {
         gens.append(",");
       }
@@ -517,7 +517,7 @@ public class LanceDbTableLsmTest {
         + shardId
         + "\",\"status\":\"Active\",\"writer_epoch\":1,\"manifest_version\":2,"
         + "\"current_generation\":9,\"replay_after_wal_entry_position\":0,"
-        + "\"wal_entry_position_last_seen\":0,\"sstables\":["
+        + "\"wal_entry_position_last_seen\":0,\"generations\":["
         + gens
         + "],\"compacting\":"
         + compacting

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.38.0-beta.11</version>
+    <version>0.38.0-beta.10</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/arrow.test.ts

@@ -1,16 +1,11 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import * as fs from "node:fs";
-import * as vm from "node:vm";
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
-  Field as CurrentField,
-  LargeBinary as CurrentLargeBinary,
-  Schema as CurrentSchema,
   Vector as CurrentVector,
   convertToTable,
   tableFromIPC as currentTableFromIPC,
@@ -41,59 +36,6 @@ function sampleRecords(): Array<Record<string, any>> {
     },
   ];
 }
-
-it("serializes an Arrow Table created in another JavaScript realm", async () => {
-  const context = vm.createContext({
-    TextDecoder,
-    TextEncoder,
-    console,
-    setTimeout,
-    clearTimeout,
-  });
-  vm.runInContext(
-    fs.readFileSync(
-      require.resolve("apache-arrow-15/Arrow.es2015.min"),
-      "utf8",
-    ),
-    context,
-  );
-  const foreignTable: unknown = vm.runInContext(
-    "Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
-    context,
-  );
-
-  const foreignMetadata = (
-    foreignTable as { schema: { metadata: Map<string, string> } }
-  ).schema.metadata;
-  expect(foreignMetadata).not.toBeInstanceOf(Map);
-
-  const buf = await fromDataToBuffer(
-    foreignTable as Parameters<typeof fromDataToBuffer>[0],
-  );
-  const actual = currentTableFromIPC(buf);
-
-  expect(actual.numRows).toBe(3);
-  expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
-  expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
-});
-
-it("preserves field metadata from a provided schema", async function () {
-  const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
-  const schema = new CurrentSchema([
-    new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
-  ]);
-
-  const table = makeArrowTable(
-    [{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
-    { schema },
-  );
-
-  expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
-
-  const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
-  expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
-});
-
 describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Arrow",
   (

nodejs/__test__/embedding.test.ts

@@ -187,58 +187,6 @@ describe("embedding functions", () => {
     const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
     expect(vector0).toEqual([1, 2, 3]);
   });
-  it("should append multiple Python embeddings with the same alias", async () => {
-    @register("python-mock")
-    // biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): Float {
-        return new Float32();
-      }
-      async computeQueryEmbeddings(_data: string) {
-        return [1, 2, 3];
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return data.map((value) =>
-          value === "hello world" ? [1, 2, 3] : [4, 5, 6],
-        );
-      }
-    }
-
-    const metadata = new Map([
-      [
-        "embedding_functions",
-        '[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
-      ],
-    ]);
-    const schema = new Schema(
-      [
-        new Field("text1", new Utf8(), true),
-        new Field("text2", new Utf8(), true),
-        new Field(
-          "vector1",
-          new FixedSizeList(3, new Field("item", new Float32(), true)),
-          true,
-        ),
-        new Field(
-          "vector2",
-          new FixedSizeList(3, new Field("item", new Float32(), true)),
-          true,
-        ),
-      ],
-      metadata,
-    );
-
-    const db = await connect(tmpDir.name);
-    const table = await db.createEmptyTable("test", schema);
-    await table.add([{ text1: "hello world", text2: "goodbye world" }]);
-
-    const rows = await table.query().toArray();
-    expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
-    expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
-  });
 
   it("should append generated vectors to a non-nullable schema", async () => {
     @register("non_nullable_schema_test")

nodejs/__test__/table.test.ts

@@ -3561,27 +3561,6 @@ describe("when creating an empty table", () => {
     expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
   });
 
-  it("can add and query JSON data", async () => {
-    const schema = new Schema([
-      new Field("id", new Int32(), true),
-      new Field(
-        "meta",
-        new Utf8(),
-        true,
-        new Map([["ARROW:extension:name", "arrow.json"]]),
-      ),
-    ]);
-    const table = await con.createEmptyTable("json", schema);
-    const meta = JSON.stringify({ x: 1 });
-
-    await table.add([{ id: 1, meta }]);
-
-    const rows = await table.query().toArray();
-    expect(rows).toHaveLength(1);
-    expect(rows[0].id).toBe(1);
-    expect(rows[0].meta).toBe(meta);
-  });
-
   it("can create an empty table from schema that specifies field types by name", async () => {
     const schemaLike = {
       fields: [

nodejs/examples/basic.test.ts

@@ -170,7 +170,7 @@ test("basic table examples", async () => {
     // --8<-- [end:create_index]
 
     // --8<-- [start:delete_rows]
-    await tbl.delete("item = 'fizz'");
+    await tbl.delete('item = "fizz"');
     // --8<-- [end:delete_rows]
 
     // --8<-- [start:drop_table]

nodejs/lancedb/arrow.ts

@@ -72,7 +72,8 @@ export type FieldLike =
     };
 
 export type DataLike =
-  | import("apache-arrow").Data
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  | import("apache-arrow").Data<Struct<any>>
   | {
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       type: any;
@@ -81,7 +82,6 @@ export type DataLike =
       stride: number;
       nullable: boolean;
       children: DataLike[];
-      dictionary?: { data: readonly DataLike[] };
       get nullCount(): number;
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       values: Buffers<any>[BufferType.DATA];

nodejs/lancedb/index.ts

@@ -157,8 +157,8 @@ export {
   TokenizeTableOptions,
   LsmWriteSpec,
   LsmStats,
-  TableShardStats,
-  SsTableStats,
+  BucketStats,
+  GenerationStats,
   MemtableStats,
   ColumnAlteration,
   FieldMetadataUpdate,

nodejs/lancedb/query.ts

@@ -727,11 +727,11 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
    * Add a query vector to the search
    *
    * This method can be called multiple times to add multiple query vectors
-   * to the search. A column called `query_index` will be added to indicate the index
-   * of the query vector that produced the result. Flat searches share one table scan
-   * across the query vectors, avoiding the scan and memory amplification of running
-   * multiple queries concurrently. Indexed searches may still perform per-vector
-   * index work.
+   * to the search. If multiple query vectors are added, then they will be searched
+   * in parallel, and the results will be concatenated. A column called `query_index`
+   * will be added to indicate the index of the query vector that produced the result.
+   *
+   * Performance wise, this is equivalent to running multiple queries concurrently.
    */
   addQueryVector(vector: IntoVector): VectorQuery {
     if (vector instanceof Promise) {

nodejs/lancedb/sanitize.ts

@@ -94,24 +94,17 @@ export function sanitizeMetadata(
   if (metadataLike === undefined || metadataLike === null) {
     return undefined;
   }
-
-  let entries: IterableIterator<[unknown, unknown]>;
-  try {
-    entries = Map.prototype.entries.call(metadataLike);
-  } catch {
+  if (!(metadataLike instanceof Map)) {
     throw Error("Expected metadata, if present, to be a Map<string, string>");
   }
-
-  const metadata = new Map<string, string>();
-  for (const [key, value] of entries) {
-    if (typeof key !== "string" || typeof value !== "string") {
+  for (const item of metadataLike) {
+    if (typeof item[0] !== "string" || typeof item[1] !== "string") {
       throw Error(
         "Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
       );
     }
-    metadata.set(key, value);
   }
-  return metadata;
+  return metadataLike as Map<string, string>;
 }
 
 export function sanitizeInt(typeLike: object) {

nodejs/lancedb/schema.ts

@@ -406,11 +406,10 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
           field.name,
           new Struct(matchingFields(struct.children, value)),
           field.nullable,
-          field.metadata,
         ),
       );
     } else {
-      matches.push(field);
+      matches.push(new Field(field.name, value as DataType, field.nullable));
     }
   }
   return matches;

nodejs/lancedb/table.ts

@@ -55,8 +55,8 @@ import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
 export {
-  TableShardStats,
-  SsTableStats,
+  BucketStats,
+  GenerationStats,
   LsmStats,
   MemtableStats,
 } from "./native";
@@ -630,18 +630,6 @@ export abstract class Table {
 
   /**
    * Update per-field (column) metadata.
-   *
-   * The following keys are treated specially, by convention, and should be
-   * used when appropriate:
-   *
-   * - `lancedb:description`: for a human-readable description of a field.
-   * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
-   *   names the tag category; e.g. `lancedb:tag:model: "clip"`.
-   * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
-   *   `feature_v2` might be in the same logical column.
-   * - `lancedb:status`: for status options (`production`, `candidate`,
-   *   `deprecated`, `archived`) to designate the current life cycle state of
-   *   this column.
    * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
    * update's metadata is merged into the field's existing metadata by default;
    * a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -741,7 +729,7 @@ export abstract class Table {
    */
   abstract closeLsmWriters(): Promise<void>;
   /**
-   * Freeze every table shard's active memtable into a new SSTable.
+   * Seal every bucket's active memtable into a new L0 generation.
    *
    * Returns once the seal is committed. Sealing an empty memtable is a no-op,
    * so this is safe to call repeatedly.
@@ -749,7 +737,7 @@ export abstract class Table {
    */
   abstract flushLsm(): Promise<void>;
   /**
-   * Trigger a background SSTable compaction pass per table shard.
+   * Trigger a background L0 → base compaction pass per bucket.
    *
    * Returns once the passes are *dispatched*, not once they finish — watch
    * {@link Table#getLsmStats} for progress, or use
@@ -760,9 +748,9 @@ export abstract class Table {
   /**
    * Converge this table's LSM write path into its base table.
    *
-   * Freezes once, then triggers compaction and polls until the SSTables that existed
+   * Seals once, then triggers compaction and polls until the L0 that existed
    * at the start is gone. The target set is fixed at the start, so
-   * SSTables created *during* the checkpoint are ignored — that is what
+   * generations created *during* the checkpoint are ignored — that is what
    * lets it terminate under write load, and what makes it best-effort: it
    * converges the fresh tier as of some instant. Idempotent, abandonable at
    * any point, and safe to run on a cadence.
@@ -786,12 +774,12 @@ export abstract class Table {
    * "why is my fresh-tier vector search brute-force". Mutates no table state.
    *
    * Resolves to `undefined` only when the LSM write path is not enabled.
-   * @param {boolean} includeSstableRows Also count rows per SSTable.
+   * @param {boolean} includeGenerationRows Also count rows per L0 generation.
    *   Off by default because each count opens an uncached Lance dataset.
    * @returns {Promise<LsmStats | undefined>}
    */
   abstract getLsmStats(
-    includeSstableRows?: boolean,
+    includeGenerationRows?: boolean,
   ): Promise<LsmStats | undefined>;
   /** Retrieve the version of the table */
 
@@ -1388,9 +1376,9 @@ export class LocalTable extends Table {
   }
 
   async getLsmStats(
-    includeSstableRows: boolean = false,
+    includeGenerationRows: boolean = false,
   ): Promise<LsmStats | undefined> {
-    return (await this.inner.getLsmStats(includeSstableRows)) ?? undefined;
+    return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
   }
 
   async version(): Promise<number> {
@@ -1567,8 +1555,7 @@ export interface FieldMetadataUpdate {
   path: string;
   /**
    * Metadata key/value pairs. Merged into the field's existing metadata by
-   * default; a value of `null` deletes that key. See
-   * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
+   * default; a value of `null` deletes that key.
    */
   metadata: Record<string, string | null>;
   /** If true, replace the field's entire metadata map instead of merging. */

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.38.0-beta.11",
+  "version": "0.38.0-beta.10",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.38.0-beta.11",
+	"version": "0.38.0-beta.10",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.38.0-beta.11",
+  "version": "0.38.0-beta.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.38.0-beta.11",
+      "version": "0.38.0-beta.10",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.38.0-beta.11",
+  "version": "0.38.0-beta.10",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/table.rs

@@ -542,11 +542,11 @@ impl Table {
     #[napi(catch_unwind)]
     pub async fn get_lsm_stats(
         &self,
-        include_sstable_rows: bool,
+        include_generation_rows: bool,
     ) -> napi::Result<Option<LsmStats>> {
         let stats = self
             .inner_ref()?
-            .get_lsm_stats(include_sstable_rows)
+            .get_lsm_stats(include_generation_rows)
             .await
             .default_error()?;
         Ok(stats.map(LsmStats::from))
@@ -950,21 +950,21 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
     }
 }
 
-/// One SSTable.
+/// One flushed L0 generation.
 #[napi(object)]
 #[derive(Clone, Debug)]
-pub struct SsTableStats {
-    /// The generation number. Increases as memtables are frozen into SSTables.
+pub struct GenerationStats {
+    /// The generation number. Increases as memtables are sealed into L0.
     pub generation: i64,
-    /// On-disk size of the SSTable.
+    /// On-disk size of the generation.
     pub bytes: i64,
-    /// Present only when `includeSstableRows` was requested. Off by default
+    /// Present only when `includeGenerationRows` was requested. Off by default
     /// because each count opens an uncached Lance dataset.
     pub rows: Option<i64>,
 }
 
-impl From<lancedb::table::SsTableStats> for SsTableStats {
-    fn from(g: lancedb::table::SsTableStats) -> Self {
+impl From<lancedb::table::GenerationStats> for GenerationStats {
+    fn from(g: lancedb::table::GenerationStats) -> Self {
         Self {
             generation: g.generation as i64,
             bytes: g.bytes as i64,
@@ -977,7 +977,7 @@ impl From<lancedb::table::SsTableStats> for SsTableStats {
 #[napi(object)]
 #[derive(Clone, Debug)]
 pub struct MemtableStats {
-    /// The generation this memtable will become once frozen.
+    /// The generation this memtable will become once sealed.
     pub generation: i64,
     /// Rows currently buffered.
     pub rows: i64,
@@ -1002,13 +1002,13 @@ impl From<lancedb::table::MemtableStats> for MemtableStats {
     }
 }
 
-/// Live state of one table shard. A table is N table shards on one node; flattening to a
-/// single number hides the one hot table shard that is usually why someone opened
+/// Live state of one bucket. A table is N buckets on one node; flattening to a
+/// single number hides the one hot bucket that is usually why someone opened
 /// this endpoint.
 #[napi(object)]
 #[derive(Clone, Debug)]
-pub struct TableShardStats {
-    /// The shard this table shard writes.
+pub struct BucketStats {
+    /// The shard this bucket writes.
     pub shard_id: String,
     /// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
     pub status: String,
@@ -1023,20 +1023,20 @@ pub struct TableShardStats {
     /// Highest WAL position the writer has seen. The difference against
     /// `replayAfterWalEntryPosition` is the WAL lag.
     pub wal_entry_position_last_seen: i64,
-    /// SSTables not yet merged into the base table.
-    pub sstables: Vec<SsTableStats>,
-    /// Whether a pass owns this table shard's compaction latch right now. Says *a*
+    /// Flushed L0 generations not yet merged into the base table.
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
     /// driver is running, not *whose*, and the latch is held from dispatch —
     /// including while the pass queues for a pod-wide compactor permit. Read it
     /// as "do not pile on", never as "mine is progressing".
     pub compacting: bool,
-    /// Oldest first, active last. Absent for a `"Sealed"` table shard, whose
+    /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
     /// in-memory state is torn down.
     pub memtables: Option<Vec<MemtableStats>>,
 }
 
-impl From<lancedb::table::TableShardStats> for TableShardStats {
-    fn from(b: lancedb::table::TableShardStats) -> Self {
+impl From<lancedb::table::BucketStats> for BucketStats {
+    fn from(b: lancedb::table::BucketStats) -> Self {
         Self {
             shard_id: b.shard_id,
             status: b.status,
@@ -1045,7 +1045,7 @@ impl From<lancedb::table::TableShardStats> for TableShardStats {
             current_generation: b.current_generation as i64,
             replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
             wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
-            sstables: b.sstables.into_iter().map(Into::into).collect(),
+            generations: b.generations.into_iter().map(Into::into).collect(),
             compacting: b.compacting,
             memtables: b
                 .memtables
@@ -1054,21 +1054,21 @@ impl From<lancedb::table::TableShardStats> for TableShardStats {
     }
 }
 
-/// Live per-table-shard LSM state, as returned by `Table#getLsmStats`.
+/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
 ///
-/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
+/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
 /// the caller's to compute.
 #[napi(object)]
 #[derive(Clone, Debug)]
 pub struct LsmStats {
-    /// One entry per table shard backing this table.
-    pub table_shards: Vec<TableShardStats>,
+    /// One entry per bucket backing this table.
+    pub buckets: Vec<BucketStats>,
 }
 
 impl From<lancedb::table::LsmStats> for LsmStats {
     fn from(stats: lancedb::table::LsmStats) -> Self {
         Self {
-            table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
+            buckets: stats.buckets.into_iter().map(Into::into).collect(),
         }
     }
 }

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

python/python/lancedb/_blob.py

@@ -270,8 +270,7 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                source = expr._column_name()
-                yield name, source if source is not None else expr.to_sql()
+                yield name, expr.to_sql()
         return
     for column in projection:
         if isinstance(column, str):
@@ -281,8 +280,7 @@ def _iter_projection_pairs(
             if isinstance(expr, str):
                 yield name, expr
             elif isinstance(expr, Expr):
-                source = expr._column_name()
-                yield name, source if source is not None else expr.to_sql()
+                yield name, expr.to_sql()
 
 
 def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:

python/python/lancedb/_lancedb.pyi

@@ -87,7 +87,6 @@ class PyExpr:
     def contains(self, substr: "PyExpr") -> "PyExpr": ...
     def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
     def cast(self, data_type: pa.DataType) -> "PyExpr": ...
-    def column_name(self) -> Optional[str]: ...
     def to_sql(self) -> str: ...
 
 def expr_col(name: str) -> PyExpr: ...
@@ -385,7 +384,7 @@ class Table:
     async def checkpoint_lsm(self) -> None: ...
     async def flush_lsm(self) -> None: ...
     async def compact_lsm(self) -> None: ...
-    async def get_lsm_stats(self, include_sstable_rows: bool) -> Optional[dict]: ...
+    async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
     async def close_lsm_writers(self) -> None: ...
     @property
     def tags(self) -> Tags: ...
@@ -609,7 +608,6 @@ class PyQueryRequest:
     filter: Optional[Union[str, bytes]]
     full_text_search: Optional[FullTextQuery]
     select: Optional[Union[str, List[str]]]
-    select_source_columns: Optional[Dict[str, str]]
     fast_search: Optional[bool]
     with_row_id: Optional[bool]
     use_lsm: Optional[bool]

python/python/lancedb/expr.py

@@ -249,10 +249,6 @@ class Expr:
 
     # ── utilities ────────────────────────────────────────────────────────────
 
-    def _column_name(self) -> str | None:
-        """Return the source name when this is a bare column expression."""
-        return self._inner.column_name()
-
     def to_sql(self) -> str:
         """Render the expression as a SQL string (useful for debugging)."""
         return self._inner.to_sql()
@@ -316,7 +312,7 @@ def func(name: str, *args: ExprLike) -> Expr:
     --------
     >>> from lancedb.expr import col, func
     >>> func("lower", col("name"))
-    Expr(lower(`name`))
+    Expr(lower(name))
     """
     inner_args = [_coerce(a)._inner for a in args]
     return Expr(expr_func(name, inner_args))

python/python/lancedb/query.py

@@ -167,12 +167,6 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
     return {"columns": projection}
 
 
-def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
-    if req.select_source_columns is not None:
-        return req.select_source_columns
-    return req.select
-
-
 def _scanner_kwargs_for_query(
     query: Query,
     blob_mode: BlobMode,
@@ -2805,16 +2799,15 @@ class AsyncQueryBase(object):
 
         req = self._inner.to_query_request()
         schema = await self._table.schema()
-        projection = _query_request_projection(req)
         self._blob_auto_row_id = blob_auto_row_id_for_scan(
             schema,
-            projection,
+            req.select,
             with_row_id=self._with_row_id,
         )
         if not self._blob_auto_row_id:
             self._blob_paths = ()
             return
-        self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
+        self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
         self._inner.with_row_id()
 
     def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3408,10 +3401,9 @@ class AsyncQuery(AsyncStandardQuery):
         pass in multiple vectors. When multiple vectors are passed in, if the vector
         column is with multivector type, then the vectors will be treated as a single
         query. Or the vectors will be treated as multiple queries, this can be useful
-        if you want to find the nearest vectors to multiple query vectors. Flat
-        searches share one table scan across the query vectors, avoiding the scan
-        and memory amplification of making multiple queries concurrently. If
-        multiple vectors are passed in then
+        if you want to find the nearest vectors to multiple query vectors.
+        This is not expected to be faster than making multiple queries concurrently;
+        it is just a convenience method. If multiple vectors are passed in then
         an additional column `query_index` will be added to the results. This column
         will contain the index of the query vector that the result is nearest to.
         """
@@ -3540,8 +3532,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
 
         Typically, a single vector is passed in as the query. However, you can also
         pass in multiple vectors.  This can be useful if you want to find the nearest
-        vectors to multiple query vectors. Flat searches share one table scan across
-        the query vectors instead of issuing concurrent full scans.
+        vectors to multiple query vectors. This is not expected to be faster than
+        making multiple queries concurrently; it is just a convenience method.
         If multiple vectors are passed in then an additional column `query_index`
         will be added to the results.  This column will contain the index of the
         query vector that the result is nearest to.
@@ -3901,15 +3893,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         blob_paths: tuple[str, ...] = ()
         if self._table is not None:
             schema = await self._table.schema()
-            projection = _query_request_projection(req)
             blob_auto_row_id = blob_auto_row_id_for_scan(
                 schema,
-                projection,
+                req.select,
                 with_row_id=self._with_row_id,
             )
             if blob_auto_row_id:
                 blob_paths = tuple(
-                    blob_v2_projection_sources(schema, projection).keys()
+                    blob_v2_projection_sources(schema, req.select).keys()
                 )
         self._blob_auto_row_id = blob_auto_row_id
         self._blob_paths = blob_paths

python/python/lancedb/remote/table.py

@@ -36,7 +36,6 @@ from lancedb._lancedb import (
     UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
-from lancedb.expr import Expr
 from lancedb.index import (
     FTS,
     BTree,
@@ -864,7 +863,7 @@ class RemoteTable(Table):
 
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -875,11 +874,9 @@ class RemoteTable(Table):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -1029,11 +1026,11 @@ class RemoteTable(Table):
         [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
         return LOOP.run(self._table.compact_lsm())
 
-    def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
+    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
         """Synchronous version of
         [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
         return LOOP.run(
-            self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows)
+            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
         )
 
     def close_lsm_writers(self) -> None:

python/python/lancedb/table.py

@@ -1744,7 +1744,7 @@ class Table(ABC):
     @abstractmethod
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -1759,11 +1759,9 @@ class Table(ABC):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -1781,7 +1779,6 @@ class Table(ABC):
         Examples
         --------
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -1791,7 +1788,7 @@ class Table(ABC):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
+        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -2130,25 +2127,12 @@ class Table(ABC):
         ----------
         updates : dict
             One or more dicts, each with:
-
             - "path": str — dot-path to the field (e.g. "embedding" or "a.b.c").
             - "metadata": dict[str, str | None] — keys to set; a value of ``None``
               deletes that key.
             - "replace": bool, optional — replace the field's whole metadata map
               instead of merging (default False).
 
-            The following keys are treated specially, by convention, and should
-            be used when appropriate:
-
-            - "lancedb:description": for a human-readable description of a field.
-            - ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
-                suffix names the tag category; e.g. "lancedb:tag:model": "clip".
-            - "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
-                and "feature_v2" might be in the same logical column.
-            - "lancedb:status" for status options ("production", "candidate",
-                "deprecated", "archived") to designate the current life cycle
-                state of this column.
-
         Returns
         -------
         UpdateFieldMetadataResult
@@ -3844,7 +3828,7 @@ class LanceTable(Table):
 
     def update(
         self,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         values: Optional[dict] = None,
         *,
         values_sql: Optional[Dict[str, str]] = None,
@@ -3855,11 +3839,9 @@ class LanceTable(Table):
 
         Parameters
         ----------
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. The filter must not be empty, or it will
-            error.
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
         values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
@@ -3877,7 +3859,6 @@ class LanceTable(Table):
         Examples
         --------
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
@@ -3887,7 +3868,7 @@ class LanceTable(Table):
         0  1  [1.0, 2.0]
         1  2  [3.0, 4.0]
         2  3  [5.0, 6.0]
-        >>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
+        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})
         UpdateResult(rows_updated=1, version=2)
         >>> table.to_pandas()
            x        vector
@@ -4189,11 +4170,11 @@ class LanceTable(Table):
         [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
         return LOOP.run(self._table.compact_lsm())
 
-    def get_lsm_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
+    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
         """Synchronous version of
         [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
         return LOOP.run(
-            self._table.get_lsm_stats(include_sstable_rows=include_sstable_rows)
+            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
         )
 
     def close_lsm_writers(self) -> None:
@@ -4916,16 +4897,16 @@ class AsyncTable:
     async def checkpoint_lsm(self) -> None:
         """Converge this table's LSM write path into its base table.
 
-        One flush, freezing every memtable into an SSTable, then compaction triggers
+        One flush, sealing every memtable into L0, then compaction triggers
         until every generation that existed at that moment has reached base.
         The loop runs client-side, reading progress from ``get_lsm_stats``.
 
-        Best-effort: SSTables created *while* it runs are deliberately not
+        Best-effort: generations created *while* it runs are deliberately not
         waited on, which is what lets it terminate on a table taking writes.
         Idempotent and safe on a cadence.
 
         There is no deadline, and the caller owns that. It returns when the
-        target SSTables are gone, raises on a terminal server fault, and
+        target generations are gone, raises on a terminal server fault, and
         otherwise waits however long the server takes. A slow table and a
         stuck one are the same picture from the client: the compactor pool is
         shared across every table on the node, so a checkpoint queued behind
@@ -4936,25 +4917,25 @@ class AsyncTable:
         await self._inner.checkpoint_lsm()
 
     async def flush_lsm(self) -> None:
-        """Freeze every table shard's active memtable into an SSTable.
+        """Seal every bucket's active memtable into L0.
 
-        Does not touch the base table — compacting SSTables into base is
+        Does not touch the base table — moving L0 into base is
         `compact_lsm`. On a node that has not claimed this table, this claims
         it and replays its WAL log first.
         """
         await self._inner.flush_lsm()
 
     async def compact_lsm(self) -> None:
-        """Trigger a background SSTable compaction pass per table shard.
+        """Trigger a background L0 to base compaction pass per bucket.
 
         Returns once the passes are dispatched, not once they finish: watch
         ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
-        until the current SSTables have reached base.
+        until the current L0 has reached base.
         """
         await self._inner.compact_lsm()
 
     async def get_lsm_stats(
-        self, *, include_sstable_rows: bool = False
+        self, *, include_generation_rows: bool = False
     ) -> Optional[dict]:
         """Read live per-bucket LSM state.
 
@@ -4967,12 +4948,12 @@ class AsyncTable:
 
         Parameters
         ----------
-        include_sstable_rows
-            Report a row count per SSTable. Off by default: each count
+        include_generation_rows
+            Report a row count per L0 generation. Off by default: each count
             opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
             needing only generation numbers.
         """
-        return await self._inner.get_lsm_stats(include_sstable_rows)
+        return await self._inner.get_lsm_stats(include_generation_rows)
 
     async def close_lsm_writers(self) -> None:
         """Drain and close any cached MemWAL shard writers for this table.
@@ -6001,7 +5982,7 @@ class AsyncTable:
         self,
         updates: Optional[Dict[str, Any]] = None,
         *,
-        where: Optional[Union[str, Expr]] = None,
+        where: Optional[str] = None,
         updates_sql: Optional[Dict[str, str]] = None,
     ) -> UpdateResult:
         """
@@ -6016,11 +5997,9 @@ class AsyncTable:
             The updates to apply.  The keys should be the name of the column to
             update.  The values should be the new values to assign.  This is
             required unless updates_sql is supplied.
-        where: str or [Expr][lancedb.expr.Expr], optional
-            The filter condition. Can be a SQL string or a type-safe
-            [Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
-            [lit][lancedb.expr.lit]. Only rows that satisfy this filter will
-            be updated.
+        where: str, optional
+            An SQL filter that controls which rows are updated. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'.  Only rows that satisfy this filter will be udpated.
         updates_sql: dict, optional
             The updates to apply, expressed as SQL expression strings.  The keys should
             be column names. The values should be SQL expressions.  These can be SQL
@@ -6038,14 +6017,13 @@ class AsyncTable:
         --------
         >>> import asyncio
         >>> import lancedb
-        >>> from lancedb.expr import col
         >>> import pandas as pd
         >>> async def demo_update():
         ...     data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
         ...     db = await lancedb.connect_async("./.lancedb")
         ...     table = await db.create_table("my_table", data)
         ...     # x is [1, 2], vector is [[1, 2], [3, 4]]
-        ...     await table.update({"vector": [10, 10]}, where=col("x") == 2)
+        ...     await table.update({"vector": [10, 10]}, where="x = 2")
         ...     # x is [1, 2], vector is [[1, 2], [10, 10]]
         ...     await table.update(updates_sql={"x": "x + 1"})
         ...     # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6059,8 +6037,7 @@ class AsyncTable:
         if updates is not None:
             updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
 
-        predicate = where.to_sql() if isinstance(where, Expr) else where
-        return await self._inner.update(updates_sql, predicate)
+        return await self._inner.update(updates_sql, where)
 
     async def add_columns(
         self,

python/python/tests/docs/test_basic.py

@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
     tbl.create_index(num_sub_vectors=1)
     # --8<-- [end:create_index]
     # --8<-- [start:delete_rows]
-    tbl.delete("item = 'fizz'")
+    tbl.delete('item = "fizz"')
     # --8<-- [end:delete_rows]
     # --8<-- [start:drop_table]
     db.drop_table("my_table")
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
     await tbl.create_index("vector")
     # --8<-- [end:create_index_async]
     # --8<-- [start:delete_rows_async]
-    await tbl.delete("item = 'fizz'")
+    await tbl.delete('item = "fizz"')
     # --8<-- [end:delete_rows_async]
     # --8<-- [start:drop_table_async]
     await db.drop_table("my_table_async")

python/python/tests/docs/test_guide_tables.py

@@ -266,7 +266,7 @@ def test_table():
     tbl.add(pydantic_model_items)
     # --8<-- [end:add_table_from_pydantic]
     # --8<-- [start:delete_row]
-    tbl.delete("item = 'fizz'")
+    tbl.delete('item = "fizz"')
     # --8<-- [end:delete_row]
     # --8<-- [start:delete_specific_row]
     data = [
@@ -538,7 +538,7 @@ async def test_table_async():
     await async_tbl.add(pydantic_model_items)
     # --8<-- [end:add_table_async_from_pydantic]
     # --8<-- [start:delete_row_async]
-    await async_tbl.delete("item = 'fizz'")
+    await async_tbl.delete('item = "fizz"')
     # --8<-- [end:delete_row_async]
     # --8<-- [start:delete_specific_row_async]
     data = [

python/python/tests/test_blob.py

@@ -8,12 +8,7 @@ import pyarrow.compute as pc
 import pytest
 
 import lancedb
-from lancedb._blob import (
-    blob_v2_projection_sources,
-    read_row_ids_from_hits,
-    stash_auto_row_ids,
-)
-from lancedb.expr import col
+from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
 from lancedb.index import FTS
 from lancedb.schema import blob_column_paths, blob_v2_column_paths
 
@@ -75,14 +70,6 @@ def test_blob_v2_column_paths_include_list_children():
     ]
 
 
-def test_blob_v2_projection_sources_use_typed_column_name():
-    schema = pa.schema([lancedb.blob("blob")])
-
-    assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
-        "blob_alias": "blob"
-    }
-
-
 def _legacy_v1_table(name):
     db = lancedb.connect("memory:///")
     schema = pa.schema(
@@ -179,20 +166,6 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
     assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
 
 
-@pytest.mark.asyncio
-async def test_async_typed_blob_projection_preserves_source_column():
-    db = await lancedb.connect_async("memory:///typed_blob_projection")
-    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
-    table = await db.create_table("typed_blob_projection", schema=schema)
-    await table.add([{"id": 1, "blob": b"alpha"}])
-
-    hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
-
-    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
-    blobs = await table.fetch_blobs("blob", hits)
-    assert blobs.to_pylist() == [b"alpha"]
-
-
 def test_fetch_blobs_round_trip():
     table = _blob_table(
         "round_trip",
@@ -430,50 +403,6 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
     assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
 
 
-@pytest.mark.asyncio
-async def test_async_hybrid_typed_blob_projection_preserves_source_column():
-    db = await lancedb.connect_async("memory:///hybrid_typed_blob")
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("text", pa.utf8()),
-            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
-            lancedb.blob("blob"),
-        ]
-    )
-    table = await db.create_table("hybrid_typed_blob", schema=schema)
-    await table.add(
-        [
-            {
-                "id": 1,
-                "text": "hello alpha",
-                "vector": [1.0, 0.0],
-                "blob": b"alpha",
-            },
-            {
-                "id": 2,
-                "text": "hello beta",
-                "vector": [0.9, 0.1],
-                "blob": b"beta",
-            },
-        ]
-    )
-    await table.create_index("text", config=FTS(with_position=False))
-
-    hits = await (
-        table.query()
-        .nearest_to([1.0, 0.0])
-        .nearest_to_text("hello")
-        .select({"blob_alias": col("blob")})
-        .limit(2)
-        .to_arrow()
-    )
-
-    assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
-    blobs = await table.fetch_blobs("blob", hits)
-    assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
-
-
 def test_blob_file_seek_read_and_read_range():
     payload = _identifiable_payload(1024)
     table = _blob_table("seek_read", [{"id": 1, "image": payload}])

python/python/tests/test_expr.py

@@ -52,7 +52,7 @@ class TestExprConstruction:
     def test_func(self):
         e = func("lower", col("name"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(`name`)"
+        assert e.to_sql() == "lower(name)"
 
     def test_func_unknown_raises(self):
         with pytest.raises(Exception):
@@ -115,7 +115,7 @@ class TestExprOperators:
     def test_and_operator(self):
         e = (col("age") > lit(18)) & (col("status") == lit("active"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"
+        assert e.to_sql() == "((age > 18) AND (status = 'active'))"
 
     def test_or_operator(self):
         e = (col("a") == lit(1)) | (col("b") == lit(2))
@@ -166,7 +166,7 @@ class TestExprOperators:
     def test_coerce_plain_str(self):
         e = col("name") == "alice"
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(`name` = 'alice')"
+        assert e.to_sql() == "(name = 'alice')"
 
     def test_reflexive_comparisons(self):
         # 10 < col("age") swaps to col("age") > 10
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:
 
     def test_bytes_equality_expr_sql(self):
         e = col("data") == lit(b"\xca\xfe")
-        assert e.to_sql() == "(`data` = X'CAFE')"
+        assert e.to_sql() == "(data = X'CAFE')"
 
     def test_bytes_ne_expr_sql(self):
         e = col("data") != lit(b"\xff")
-        assert e.to_sql() == "(`data` <> X'FF')"
+        assert e.to_sql() == "(data <> X'FF')"
 
     def test_bytes_compound_expr_sql(self):
         e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
-        assert e.to_sql() == "((`data` = X'01') AND (id > 5))"
+        assert e.to_sql() == "((data = X'01') AND (id > 5))"
 
     def test_bytes_in_function_call(self):
         # Regression test: binary literals inside scalar function calls
         # used to fail because DataFusion's unparser does not support Binary
         # scalars.  Now handled via a placeholder-substitution rewrite.
         e = func("contains", col("data"), lit(b"\xff"))
-        assert e.to_sql() == "contains(`data`, X'FF')"
+        assert e.to_sql() == "contains(data, X'FF')"
 
     def test_bytes_in_not(self):
         e = ~(col("data") == lit(b"\xff"))
-        assert e.to_sql() == "NOT (`data` = X'FF')"
+        assert e.to_sql() == "NOT (data = X'FF')"
 
 
 class TestExprStringMethods:
     def test_lower(self):
         e = col("name").lower()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "lower(`name`)"
+        assert e.to_sql() == "lower(name)"
 
     def test_upper(self):
         e = col("name").upper()
         assert isinstance(e, Expr)
-        assert e.to_sql() == "upper(`name`)"
+        assert e.to_sql() == "upper(name)"
 
     def test_contains(self):
         e = col("text").contains(lit("hello"))
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(`text`, 'hello')"
+        assert e.to_sql() == "contains(text, 'hello')"
 
     def test_contains_with_str_coerce(self):
         e = col("text").contains("hello")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "contains(`text`, 'hello')"
+        assert e.to_sql() == "contains(text, 'hello')"
 
     def test_chained_lower_eq(self):
         e = col("name").lower() == lit("alice")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "(lower(`name`) = 'alice')"
+        assert e.to_sql() == "(lower(name) = 'alice')"
 
 
 class TestExprCast:
     def test_cast_string(self):
         e = col("id").cast("string")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
+        assert e.to_sql() == "CAST(id AS VARCHAR)"
 
     def test_cast_int32(self):
         e = col("score").cast("int32")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(score, 'Int32')"
+        assert e.to_sql() == "CAST(score AS INTEGER)"
 
     def test_cast_float64(self):
         e = col("val").cast("float64")
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(val, 'Float64')"
+        assert e.to_sql() == "CAST(val AS DOUBLE)"
 
     def test_cast_pyarrow_type(self):
         e = col("score").cast(pa.int32())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(score, 'Int32')"
+        assert e.to_sql() == "CAST(score AS INTEGER)"
 
     def test_cast_pyarrow_float64(self):
         e = col("val").cast(pa.float64())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(val, 'Float64')"
+        assert e.to_sql() == "CAST(val AS DOUBLE)"
 
     def test_cast_pyarrow_string(self):
         e = col("id").cast(pa.string())
         assert isinstance(e, Expr)
-        assert e.to_sql() == "arrow_cast(id, 'Utf8')"
+        assert e.to_sql() == "CAST(id AS VARCHAR)"
 
     def test_cast_pyarrow_and_string_equivalent(self):
         # pa.int32() and "int32" should produce equivalent SQL
@@ -597,14 +597,14 @@ class TestExprIsin:
     def test_isin_strs(self):
         assert (
             col("status").isin(["active", "pending"]).to_sql()
-            == "`status` IN ('active', 'pending')"
+            == "status IN ('active', 'pending')"
         )
 
     def test_isin_coerces_and_mixes(self):
         assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"
 
     def test_isin_empty(self):
-        assert col("id").isin([]).to_sql() == "false"
+        assert col("id").isin([]).to_sql() == "id IN ()"
 
     def test_isin_filter(self, simple_table):
         result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()

python/python/tests/test_query.py

@@ -675,21 +675,6 @@ def test_distance_range(table: lancedb.table.Table):
     assert res["_distance"].to_pylist() == [min_dist, max_dist]
 
 
-@pytest.mark.parametrize("expression", ["1 - _distance", "1.0 - _distance"])
-def test_select_arithmetic_with_distance(table, expression):
-    result = (
-        table.search([10, 10])
-        .select({"similarity": expression, "_distance": "_distance"})
-        .distance_type("cosine")
-        .to_arrow()
-    )
-
-    assert result.schema.field("similarity").type == pa.float32()
-    assert result["similarity"].to_pylist() == pytest.approx(
-        [1 - distance for distance in result["_distance"].to_pylist()]
-    )
-
-
 @pytest.mark.asyncio
 async def test_distance_range_async(table_async: AsyncTable):
     q = [0, 0]
@@ -912,23 +897,6 @@ def test_query_builder_batches(table):
     assert rs_list["id"][1] == 2
 
 
-def test_batch_vector_query_shares_filtered_flat_scan(table):
-    query = (
-        table.search([[1.0, 2.0], [3.0, 4.0]])
-        .where("id > 0", prefilter=True)
-        .limit(1)
-        .select(["id"])
-    )
-
-    plan = query.explain_plan(verbose=True)
-    assert "KNNVectorDistance: queries=2" in plan
-    assert "UnionExec" not in plan
-
-    results = query.to_arrow()
-    assert len(results) == 2
-    assert results["query_index"].to_pylist() == [0, 1]
-
-
 def test_dynamic_projection(table):
     rs = (
         LanceVectorQueryBuilder(table, [0, 0], "vector")

python/python/tests/test_remote_db.py

@@ -1278,9 +1278,9 @@ def test_get_lsm_stats_sync():
     with lsm_test_table(lsm_handler) as table:
         assert table.get_lsm_stats() == {"buckets": [bucket]}
         # Off by default, and forwarded when asked for.
-        assert seen_bodies == [{"include_sstable_rows": False}]
-        table.get_lsm_stats(include_sstable_rows=True)
-        assert seen_bodies[-1] == {"include_sstable_rows": True}
+        assert seen_bodies == [{"include_generation_rows": False}]
+        table.get_lsm_stats(include_generation_rows=True)
+        assert seen_bodies[-1] == {"include_generation_rows": True}
 
 
 def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
@@ -1309,7 +1309,7 @@ def test_flush_and_compact_lsm_sync():
 
 
 def test_checkpoint_lsm_sync():
-    """Freeze, read the watermark, and return once no SSTables remain.
+    """Seal, read the watermark, and return once L0 holds nothing.
 
     The convergence loop itself is covered in Rust; this pins the sync
     binding to the endpoints it drives.
@@ -1319,7 +1319,7 @@ def test_checkpoint_lsm_sync():
     def lsm_handler(request, route):
         called.append(route)
         if route == "get_lsm_stats":
-            # An empty SSTable tier yields no target watermark, so the loop is done
+            # An empty L0 yields no target watermark, so the loop is done
             # after the seal without ever polling compaction.
             send_json(request, {"lsm_stats": {"buckets": []}})
         else:

python/python/tests/test_table.py

@@ -11,7 +11,6 @@ import warnings
 import weakref
 from concurrent.futures import ThreadPoolExecutor
 from datetime import date, datetime, timedelta
-from decimal import Decimal
 from time import sleep
 from typing import List
 from unittest.mock import patch
@@ -337,21 +336,6 @@ async def test_update_async(mem_db_async: AsyncConnection):
     assert await table.count_rows("id == 10") == 1
 
 
-@pytest.mark.asyncio
-async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
-    values = ["5", "4.66e-84", "it's"]
-    table = await mem_db_async.create_table(
-        "update_expr_literals",
-        data=[{"field": value, "result": "original"} for value in values],
-    )
-
-    for value in values:
-        update_res = await table.update({"result": value}, where=col("field") == value)
-        assert update_res.rows_updated == 1
-
-    assert (await table.to_arrow())["result"].to_pylist() == values
-
-
 def test_create_table(mem_db: DBConnection):
     schema = pa.schema(
         {
@@ -2359,148 +2343,6 @@ def test_update(mem_db: DBConnection):
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
-def test_update_expr_filter_literals(mem_db: DBConnection):
-    values = ["5", "4.66e-84", "it's"]
-    table = mem_db.create_table(
-        "update_expr_literals",
-        data=[{"field": value, "result": "original"} for value in values],
-    )
-
-    for value in values:
-        update_res = table.update(where=col("field") == value, values={"result": value})
-        assert update_res.rows_updated == 1
-
-    assert table.to_arrow()["result"].to_pylist() == values
-
-
-def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
-    low = Decimal("1.234567890123456789")
-    high = Decimal("1.234567890123456790")
-    decimal_schema = pa.schema(
-        [("val", pa.decimal128(19, 18)), ("result", pa.string())]
-    )
-    decimal_table = mem_db.create_table(
-        "update_expr_decimal",
-        pa.table(
-            {"val": [low, high], "result": ["old", "old"]},
-            schema=decimal_schema,
-        ),
-    )
-    predicate = col("val") < lit(high)
-    assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
-    result = decimal_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    keyword_table = mem_db.create_table(
-        "update_expr_keyword", [{"null": 1, "result": "old"}]
-    )
-    predicate = col("null") == 1
-    assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
-    result = keyword_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    empty_in_table = mem_db.create_table(
-        "update_expr_empty_in", [{"id": 1, "result": "old"}]
-    )
-    predicate = col("id").isin([])
-    assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
-    result = empty_in_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 0
-
-    marker = "__lancedb_binary_placeholder_0__"
-    binary_schema = pa.schema(
-        [("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
-    )
-    binary_table = mem_db.create_table(
-        "update_expr_binary",
-        pa.table(
-            {
-                "payload": [b"\x01", b"\x02"],
-                "text": ["other", marker],
-                "result": ["old", "old"],
-            },
-            schema=binary_schema,
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
-    assert binary_table.search().where(predicate).to_arrow().num_rows == 2
-    result = binary_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 2
-
-    nonfinite_table = mem_db.create_table(
-        "update_expr_nonfinite",
-        [{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
-    )
-    predicate = col("x") < float("inf")
-    assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
-    result = nonfinite_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 2
-
-    float16_table = mem_db.create_table(
-        "update_expr_float16",
-        [{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
-    )
-    predicate = col("x").cast(pa.float16()) < 2.0
-    assert float16_table.search().where(predicate).to_arrow().num_rows == 1
-    result = float16_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    string_cast_table = mem_db.create_table(
-        "update_expr_string_cast",
-        [{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
-    )
-    predicate = col("x").cast("string") == "1"
-    assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
-    result = string_cast_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    quoted_identifier_schema = pa.schema(
-        [("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
-    )
-    quoted_identifier_table = mem_db.create_table(
-        "update_expr_quoted_identifier",
-        pa.table(
-            {"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
-            schema=quoted_identifier_schema,
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
-    assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
-    result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    decimal256_schema = pa.schema(
-        [("val", pa.decimal256(40, 2)), ("result", pa.string())]
-    )
-    decimal256_table = mem_db.create_table(
-        "update_expr_decimal256",
-        pa.table(
-            {
-                "val": [Decimal("1.00"), Decimal("3.00")],
-                "result": ["old", "old"],
-            },
-            schema=decimal256_schema,
-        ),
-    )
-    predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
-    assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
-    result = decimal256_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 1
-
-    binary_empty_table = mem_db.create_table(
-        "update_expr_binary_empty",
-        pa.table(
-            {"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
-            schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
-        ),
-    )
-    predicate = (col("payload") == lit(b"\x01")).isin([])
-    assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
-    assert predicate.to_sql() == "false"
-    result = binary_empty_table.update(where=predicate, values={"result": "new"})
-    assert result.rows_updated == 0
-
-
 def test_update_with_arrow_scalar(mem_db: DBConnection):
     schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
     table = mem_db.create_table("my_table", schema=schema)

python/src/expr.rs

@@ -130,14 +130,6 @@ impl PyExpr {
 
     // ── utilities ────────────────────────────────────────────────────────────
 
-    /// Return the referenced column name for a bare column expression.
-    fn column_name(&self) -> Option<String> {
-        match &self.0 {
-            DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()),
-            _ => None,
-        }
-    }
-
     /// Render the expression as a SQL string (useful for debugging).
     fn to_sql(&self) -> PyResult<String> {
         lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string()))

python/src/query.rs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -326,7 +325,6 @@ pub struct PyQueryRequest {
     pub filter: Option<PyQueryFilter>,
     pub full_text_search: Option<PyLanceDB<FtsQuery>>,
     pub select: PySelect,
-    pub select_source_columns: Option<HashMap<String, String>>,
     pub fast_search: Option<bool>,
     pub with_row_id: Option<bool>,
     pub use_lsm: Option<bool>,
@@ -357,7 +355,6 @@ impl From<AnyQuery> for PyQueryRequest {
                 full_text_search: query_request
                     .full_text_search
                     .map(|fts| PyLanceDB(fts.query)),
-                select_source_columns: PySelect::source_columns(&query_request.select),
                 select: PySelect(query_request.select),
                 fast_search: Some(query_request.fast_search),
                 with_row_id: Some(query_request.with_row_id),
@@ -383,7 +380,6 @@ impl From<AnyQuery> for PyQueryRequest {
                 offset: vector_query.base.offset,
                 filter: vector_query.base.filter.map(PyQueryFilter),
                 full_text_search: None,
-                select_source_columns: PySelect::source_columns(&vector_query.base.select),
                 select: PySelect(vector_query.base.select),
                 fast_search: Some(vector_query.base.fast_search),
                 with_row_id: Some(vector_query.base.with_row_id),
@@ -416,25 +412,6 @@ impl From<AnyQuery> for PyQueryRequest {
 #[derive(Clone)]
 pub struct PySelect(Select);
 
-impl PySelect {
-    fn source_columns(select: &Select) -> Option<HashMap<String, String>> {
-        match select {
-            Select::Expr(pairs) => Some(
-                pairs
-                    .iter()
-                    .filter_map(|(output, expr)| match expr {
-                        lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => {
-                            Some((output.clone(), column.name.clone()))
-                        }
-                        _ => None,
-                    })
-                    .collect(),
-            ),
-            _ => None,
-        }
-    }
-}
-
 impl<'py> IntoPyObject<'py> for PySelect {
     type Target = PyAny;
     type Output = Bound<'py, Self::Target>;

python/src/table.rs

@@ -33,16 +33,16 @@ use pyo3::{
 
 mod scannable;
 
-/// Convert `LsmStats` to a Python dict, preserving the per-table-shard list.
+/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
 ///
 /// Deliberately not flattened to a table-level summary: a table is N
-/// table shards on one node, and the per-shard detail is the reason the
-/// endpoint exists — flattening hides the single hot table shard someone opened
+/// buckets on one node, and the per-bucket detail is the reason the
+/// endpoint exists — flattening hides the single hot bucket someone opened
 /// it to find.
 fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
     let out = PyDict::new(py);
-    let table_shards = PyList::empty(py);
-    for b in &stats.table_shards {
+    let buckets = PyList::empty(py);
+    for b in &stats.buckets {
         let e = PyDict::new(py);
         e.set_item("shard_id", &b.shard_id)?;
         e.set_item("status", &b.status)?;
@@ -58,15 +58,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
             b.wal_entry_position_last_seen,
         )?;
 
-        let sstables = PyList::empty(py);
-        for g in &b.sstables {
+        let generations = PyList::empty(py);
+        for g in &b.generations {
             let ge = PyDict::new(py);
             ge.set_item("generation", g.generation)?;
             ge.set_item("bytes", g.bytes)?;
             ge.set_item("rows", g.rows)?;
-            sstables.append(ge)?;
+            generations.append(ge)?;
         }
-        e.set_item("sstables", sstables)?;
+        e.set_item("generations", generations)?;
         e.set_item("compacting", b.compacting)?;
 
         e.set_item(
@@ -88,9 +88,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
                 })
                 .transpose()?,
         )?;
-        table_shards.append(e)?;
+        buckets.append(e)?;
     }
-    out.set_item("table_shards", table_shards)?;
+    out.set_item("buckets", buckets)?;
     Ok(out.unbind())
 }
 
@@ -1492,7 +1492,7 @@ impl Table {
         })
     }
 
-    /// Freeze every table shard's active memtable into an SSTable.
+    /// Seal every bucket's active memtable into L0.
     pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(
@@ -1501,7 +1501,7 @@ impl Table {
         )
     }
 
-    /// Trigger a background SSTable compaction pass per table shard. Returns once the
+    /// Trigger a background L0 → base pass per bucket. Returns once the
     /// passes are dispatched, not once they finish — watch `get_lsm_stats`.
     pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
@@ -1511,15 +1511,15 @@ impl Table {
     }
 
     /// Live LSM state, or `None` when the LSM write path is not enabled.
-    #[pyo3(signature = (include_sstable_rows=false))]
+    #[pyo3(signature = (include_generation_rows=false))]
     pub fn get_lsm_stats(
         self_: PyRef<'_, Self>,
-        include_sstable_rows: bool,
+        include_generation_rows: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let stats = inner
-                .get_lsm_stats(include_sstable_rows)
+                .get_lsm_stats(include_generation_rows)
                 .await
                 .infer_error()?;
             Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.38.0-beta.11"
+version = "0.38.0-beta.10"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/lancedb/src/expr.rs

@@ -19,7 +19,6 @@
 
 mod sql;
 
-pub(crate) use sql::canonicalize_sql_predicate;
 pub use sql::expr_to_sql_string;
 
 use std::sync::Arc;
@@ -157,7 +156,7 @@ mod tests {
         use datafusion_common::ScalarValue;
         let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "(`data` = X'CAFE')");
+        assert_eq!(sql, "(data = X'CAFE')");
     }
 
     #[test]
@@ -167,7 +166,7 @@ mod tests {
         let int_expr = col("id").gt(lit(5i64));
         let combined = bin_expr.and(int_expr);
         let sql = expr_to_sql_string(&combined).unwrap();
-        assert_eq!(sql, "((`data` = X'01') AND (id > 5))");
+        assert_eq!(sql, "((data = X'01') AND (id > 5))");
     }
 
     #[test]
@@ -185,7 +184,7 @@ mod tests {
         // serialized correctly (regression test for placeholder rewrite path).
         let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff]))));
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "contains(`data`, X'FF')");
+        assert_eq!(sql, "contains(data, X'FF')");
     }
 
     #[test]
@@ -196,7 +195,7 @@ mod tests {
             .eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd]))))
             .not();
         let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(sql, "NOT (`data` = X'ABCD')");
+        assert_eq!(sql, "NOT (data = X'ABCD')");
     }
 
     #[test]
@@ -206,122 +205,6 @@ mod tests {
         assert!(sql.contains("IN"), "expected IN in: {}", sql);
     }
 
-    #[test]
-    fn test_empty_is_in() {
-        let expr = is_in(col("id"), vec![]);
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
-    }
-
-    #[test]
-    fn test_empty_is_in_discards_binary_children() {
-        use datafusion_common::ScalarValue;
-
-        let expr = is_in(
-            col("payload").eq(lit(ScalarValue::Binary(Some(vec![0x01])))),
-            vec![],
-        );
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
-    }
-
-    #[test]
-    fn test_keyword_identifier() {
-        let expr = col("null").eq(lit(1i64));
-        assert_eq!(expr_to_sql_string(&expr).unwrap(), "(`null` = 1)");
-    }
-
-    #[test]
-    fn test_decimal_literal_preserves_type() {
-        use datafusion_common::ScalarValue;
-
-        let expr = col("val").lt(lit(ScalarValue::Decimal128(
-            Some(1_234_567_890_123_456_790),
-            19,
-            18,
-        )));
-        let sql = expr_to_sql_string(&expr).unwrap();
-        assert_eq!(
-            sql,
-            "(val < arrow_cast('1.234567890123456790', 'Decimal128(19, 18)'))"
-        );
-    }
-
-    #[test]
-    fn test_non_finite_float_literal_preserves_type() {
-        let expr = col("x").lt(lit(f64::INFINITY));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(x < arrow_cast('inf', 'Float64'))"
-        );
-    }
-
-    #[test]
-    fn test_cast_uses_arrow_type_name() {
-        let string = expr_cast(col("x"), DataType::Utf8);
-        assert_eq!(
-            expr_to_sql_string(&string).unwrap(),
-            "arrow_cast(x, 'Utf8')"
-        );
-
-        let int32 = expr_cast(col("x"), DataType::Int32);
-        assert_eq!(
-            expr_to_sql_string(&int32).unwrap(),
-            "arrow_cast(x, 'Int32')"
-        );
-
-        let expr = expr_cast(col("x"), DataType::Float16).lt(lit(2.0));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(arrow_cast(x, 'Float16') < 2.0)"
-        );
-
-        let decimal = expr_cast(lit("2.00"), DataType::Decimal256(40, 2));
-        assert_eq!(
-            expr_to_sql_string(&decimal).unwrap(),
-            "arrow_cast('2.00', 'Decimal256(40, 2)')"
-        );
-    }
-
-    #[test]
-    fn test_binary_placeholder_does_not_rewrite_user_string() {
-        use datafusion_common::ScalarValue;
-
-        let marker = "__lancedb_binary_placeholder_0__";
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .or(col("text").eq(lit(marker)));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "((payload = X'01') OR (`text` = '__lancedb_binary_placeholder_0__'))"
-        );
-    }
-
-    #[test]
-    fn test_binary_binding_skips_quoted_identifiers() {
-        use datafusion_common::ScalarValue;
-
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .and(col("odd'name").eq(lit(1i64)))
-            .and(col("odd`'name").eq(lit(2i64)));
-        assert_eq!(
-            expr_to_sql_string(&expr).unwrap(),
-            "(((payload = X'01') AND (`odd'name` = 1)) AND (`odd``'name` = 2))"
-        );
-    }
-
-    #[test]
-    fn test_binary_placeholder_collision_search_is_linear() {
-        use datafusion_common::ScalarValue;
-
-        let collision_shaped = format!("__lancedb_binary_placeholder_0__{}", "_".repeat(64_000));
-        let expr = col("payload")
-            .eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
-            .and(col("text").eq(lit(collision_shaped.clone())));
-        let sql = expr_to_sql_string(&expr).unwrap();
-        assert!(sql.contains("X'01'"));
-        assert!(sql.contains(&format!("'{collision_shaped}'")));
-    }
-
     #[test]
     fn test_multiple_binary_literals() {
         use datafusion_common::ScalarValue;

rust/lancedb/src/expr/sql.rs

@@ -1,27 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::{
-    any::TypeId,
-    collections::{HashMap, HashSet},
-};
-
-use arrow_array::types::{
-    Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
-};
-use arrow_schema::DataType;
 use datafusion_common::ScalarValue;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
 use datafusion_expr::Expr;
-use datafusion_functions::core::expr_fn::{
-    arrow_cast as datafusion_arrow_cast, arrow_try_cast as datafusion_arrow_try_cast,
-};
-use datafusion_sql::sqlparser::{
-    dialect::{Dialect as SqlParserDialect, GenericDialect},
-    keywords::ALL_KEYWORDS,
-    tokenizer::{Token, Tokenizer},
-};
-use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
+use datafusion_sql::unparser::{self, dialect::Dialect};
 
 /// Unparser dialect that matches the quoting style expected by the Lance SQL
 /// parser.  Lance uses backtick (`` ` ``) as the only delimited-identifier
@@ -36,74 +19,17 @@ use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
 ///   lower-case by the SQL parser, which would break case-sensitive schemas).
 struct LanceSqlDialect;
 
-impl UnparserDialect for LanceSqlDialect {
+impl Dialect for LanceSqlDialect {
     fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
-        let identifier_upper = identifier.to_ascii_uppercase();
-        let needs_quote =
-            (identifier_upper != "ID" && ALL_KEYWORDS.contains(&identifier_upper.as_str()))
-                || identifier.chars().any(|c| c.is_ascii_uppercase())
-                || !identifier.chars().enumerate().all(|(i, c)| {
-                    c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
-                });
+        let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
+            || !identifier
+                .chars()
+                .enumerate()
+                .all(|(i, c)| c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()));
         if needs_quote { Some('`') } else { None }
     }
 }
 
-/// Lance's tokenizer dialect with SQL-standard double-quoted identifiers added.
-///
-/// Keep this deliberately small: Lance's parser wraps `GenericDialect` and
-/// delegates only identifier recognition, leaving every other dialect option at
-/// its default. In particular, `/*! ... */` remains an ordinary block comment.
-#[derive(Debug, Default)]
-struct PredicateDialect(GenericDialect);
-
-impl SqlParserDialect for PredicateDialect {
-    fn dialect(&self) -> TypeId {
-        self.0.dialect()
-    }
-
-    fn is_identifier_start(&self, ch: char) -> bool {
-        self.0.is_identifier_start(ch)
-    }
-
-    fn is_identifier_part(&self, ch: char) -> bool {
-        self.0.is_identifier_part(ch)
-    }
-
-    fn is_delimited_identifier_start(&self, ch: char) -> bool {
-        ch == '"' || ch == '`'
-    }
-}
-
-/// Canonicalize a raw SQL predicate for Lance's parser.
-///
-/// Lance wraps [`GenericDialect`] for identifier recognition while retaining the
-/// default dialect behavior for every other lexical option. [`PredicateDialect`]
-/// mirrors that contract and additionally recognizes `"` as an identifier
-/// delimiter, allowing this function to rewrite only those identifier tokens.
-pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result<String> {
-    let dialect = PredicateDialect::default();
-    let tokens = Tokenizer::new(&dialect, predicate)
-        .with_unescape(false)
-        .tokenize()
-        .map_err(|err| crate::Error::InvalidInput {
-            message: format!("invalid SQL predicate: {err}"),
-        })?;
-
-    Ok(tokens
-        .into_iter()
-        .map(|token| match token {
-            Token::Word(word) if word.quote_style == Some('"') => {
-                // with_unescape(false) retains doubled double quotes. Decode
-                // those before escaping any backticks for Lance's delimiter.
-                let identifier = word.value.replace("\"\"", "\"").replace('`', "``");
-                format!("`{identifier}`")
-            }
-            other => other.to_string(),
-        })
-        .collect())
-}
-
 /// Prefix for placeholder strings inserted in place of binary literals.  Chosen
 /// to be extremely unlikely to occur in user data.
 const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
@@ -113,128 +39,24 @@ fn bytes_to_hex_sql(bytes: &[u8]) -> String {
     format!("X'{hex}'")
 }
 
-fn string_literals(expr: &Expr) -> HashSet<String> {
-    let mut literals = HashSet::new();
+/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal
+/// anywhere in its subtree.  DataFusion's SQL unparser cannot serialize those
+/// variants, so we route such expressions through a placeholder-substitution
+/// path that emits SQL `X'...'` byte-string literals.
+fn has_binary_literal(expr: &Expr) -> bool {
+    let mut found = false;
     let _ = expr.apply(&mut |e: &Expr| {
-        if let Expr::Literal(
-            ScalarValue::Utf8(Some(value))
-            | ScalarValue::LargeUtf8(Some(value))
-            | ScalarValue::Utf8View(Some(value)),
-            _,
-        ) = e
-        {
-            literals.insert(value.clone());
-        }
-        Ok(TreeNodeRecursion::Continue)
-    });
-    literals
-}
-
-fn typed_string_literal(value: String, data_type: DataType) -> Expr {
-    datafusion_arrow_cast(
-        Expr::Literal(ScalarValue::Utf8(Some(value)), None),
-        Expr::Literal(ScalarValue::Utf8(Some(data_type.to_string())), None),
-    )
-}
-
-fn next_binary_placeholder(user_strings: &HashSet<String>, next_id: &mut usize) -> String {
-    loop {
-        let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id);
-        *next_id += 1;
-        if !user_strings.contains(&placeholder) {
-            return placeholder;
-        }
-    }
-}
-
-fn bind_binary_literals(
-    sql: &str,
-    mut bindings: HashMap<String, Vec<u8>>,
-) -> crate::Result<String> {
-    let bytes = sql.as_bytes();
-    let mut output = Vec::with_capacity(bytes.len());
-    let mut index = 0;
-
-    // Walk SQL string tokens once. Placeholders are plain, unescaped string
-    // literals, so this remains linear even when user strings are large or
-    // deliberately resemble the placeholder prefix.
-    while index < bytes.len() {
-        if bytes[index] == b'`' {
-            let identifier_start = index;
-            index += 1;
-            let mut identifier_end = None;
-            while index < bytes.len() {
-                if bytes[index] == b'`' {
-                    if index + 1 < bytes.len() && bytes[index + 1] == b'`' {
-                        index += 2;
-                    } else {
-                        index += 1;
-                        identifier_end = Some(index);
-                        break;
-                    }
-                } else {
-                    index += 1;
-                }
-            }
-
-            let Some(identifier_end) = identifier_end else {
-                return Err(crate::Error::InvalidInput {
-                    message: "unterminated identifier while binding binary literal".to_string(),
-                });
-            };
-            output.extend_from_slice(&bytes[identifier_start..identifier_end]);
-            continue;
-        }
-
-        if bytes[index] != b'\'' {
-            output.push(bytes[index]);
-            index += 1;
-            continue;
-        }
-
-        let literal_start = index;
-        index += 1;
-        let content_start = index;
-        let mut escaped = false;
-        let mut content_end = None;
-        while index < bytes.len() {
-            if bytes[index] == b'\'' {
-                if index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
-                    escaped = true;
-                    index += 2;
-                } else {
-                    content_end = Some(index);
-                    index += 1;
-                    break;
-                }
-            } else {
-                index += 1;
-            }
-        }
-
-        let Some(content_end) = content_end else {
-            return Err(crate::Error::InvalidInput {
-                message: "unterminated string while binding binary literal".to_string(),
-            });
-        };
-
-        let placeholder = &sql[content_start..content_end];
-        if !escaped && let Some(value) = bindings.remove(placeholder) {
-            output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes());
+        if matches!(
+            e,
+            Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _)
+        ) {
+            found = true;
+            Ok(TreeNodeRecursion::Stop)
         } else {
-            output.extend_from_slice(&bytes[literal_start..index]);
+            Ok(TreeNodeRecursion::Continue)
         }
-    }
-
-    if !bindings.is_empty() {
-        return Err(crate::Error::InvalidInput {
-            message: "failed to bind binary literal while serializing expression".to_string(),
-        });
-    }
-
-    String::from_utf8(output).map_err(|e| crate::Error::InvalidInput {
-        message: format!("failed to bind binary literal: {e}"),
-    })
+    });
+    found
 }
 
 fn run_unparser(expr: &Expr) -> crate::Result<String> {
@@ -247,37 +69,25 @@ fn run_unparser(expr: &Expr) -> crate::Result<String> {
 }
 
 pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
-    // DataFusion's unparser needs a few adaptations before its SQL can be
-    // reparsed by Lance without changing the typed expression's semantics:
-    //
-    // * decimal literals need an explicit cast to preserve precision and scale;
-    // * casts need exact Arrow type names rather than SQL type aliases;
-    // * an empty IN list is valid in DataFusion but invalid SQL;
-    // * binary literals are unsupported by the unparser and need placeholders.
-    // Eliminate empty membership expressions before visiting their children.
-    // Otherwise a discarded binary child could leave behind a stale binding.
+    // Fast path: no binary literals — DataFusion's unparser handles everything.
+    if !has_binary_literal(expr) {
+        return run_unparser(expr);
+    }
+
+    // Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary`
+    // scalars, so we rewrite each one to a unique string-literal placeholder,
+    // let the unparser do the rest of the work, then substitute the SQL
+    // `X'...'` byte-string literal back in.  This keeps the operator/function
+    // serialization logic centralized in DataFusion and works for every
+    // expression node type the unparser supports.
+    let mut bindings: Vec<Vec<u8>> = Vec::new();
     let rewritten = expr
         .clone()
-        .transform(|e: Expr| match e {
-            Expr::InList(in_list) if in_list.list.is_empty() => Ok(Transformed::yes(
-                Expr::Literal(ScalarValue::Boolean(Some(in_list.negated)), None),
-            )),
-            other => Ok(Transformed::no(other)),
-        })
-        .map_err(|e| crate::Error::InvalidInput {
-            message: format!("failed to rewrite expression: {e}"),
-        })?
-        .data;
-
-    let user_strings = string_literals(&rewritten);
-    let mut next_placeholder_id = 0;
-    let mut binary_bindings = HashMap::new();
-    let rewritten = rewritten
         .transform(|e: Expr| match e {
             Expr::Literal(ScalarValue::Binary(Some(bytes)), m)
             | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => {
-                let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id);
-                binary_bindings.insert(placeholder.clone(), bytes);
+                let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len());
+                bindings.push(bytes);
                 Ok(Transformed::yes(Expr::Literal(
                     ScalarValue::Utf8(Some(placeholder)),
                     m,
@@ -287,57 +97,6 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
             | Expr::Literal(ScalarValue::LargeBinary(None), m) => {
                 Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m)))
             }
-            Expr::Literal(ScalarValue::Decimal32(Some(value), precision, scale), _m) => {
-                let value = Decimal32Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal32(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal64(Some(value), precision, scale), _m) => {
-                let value = Decimal64Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal64(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal128(Some(value), precision, scale), _m) => {
-                let value = Decimal128Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal128(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Decimal256(Some(value), precision, scale), _m) => {
-                let value = Decimal256Type::format_decimal(value, precision, scale);
-                Ok(Transformed::yes(typed_string_literal(
-                    value,
-                    DataType::Decimal256(precision, scale),
-                )))
-            }
-            Expr::Literal(ScalarValue::Float16(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float16)),
-            ),
-            Expr::Literal(ScalarValue::Float32(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float32)),
-            ),
-            Expr::Literal(ScalarValue::Float64(Some(value)), _m) if !value.is_finite() => Ok(
-                Transformed::yes(typed_string_literal(value.to_string(), DataType::Float64)),
-            ),
-            Expr::Cast(cast) => Ok(Transformed::yes(datafusion_arrow_cast(
-                *cast.expr,
-                Expr::Literal(
-                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
-                    None,
-                ),
-            ))),
-            Expr::TryCast(cast) => Ok(Transformed::yes(datafusion_arrow_try_cast(
-                *cast.expr,
-                Expr::Literal(
-                    ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
-                    None,
-                ),
-            ))),
             other => Ok(Transformed::no(other)),
         })
         .map_err(|e| crate::Error::InvalidInput {
@@ -345,58 +104,12 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
         })?
         .data;
 
-    let sql = run_unparser(&rewritten)?;
-    if binary_bindings.is_empty() {
-        Ok(sql)
-    } else {
-        bind_binary_literals(&sql, binary_bindings)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::canonicalize_sql_predicate;
-
-    #[test]
-    fn normalizes_double_quoted_identifiers() {
-        assert_eq!(
-            canonicalize_sql_predicate(r#""PartyAbbrev" = 'D'"#).unwrap(),
-            "`PartyAbbrev` = 'D'"
-        );
-        assert_eq!(
-            canonicalize_sql_predicate(r#""MetaData"."userId" = 5"#).unwrap(),
-            "`MetaData`.`userId` = 5"
-        );
-        assert_eq!(
-            canonicalize_sql_predicate(r#""a""b" = 1"#).unwrap(),
-            "`a\"b` = 1"
-        );
-    }
-
-    #[test]
-    fn preserves_quotes_inside_literals_and_backticks() {
-        let filter = r#"name = 'Alice "Ace"' AND `quoted"field` = 1"#;
-        assert_eq!(canonicalize_sql_predicate(filter).unwrap(), filter);
-    }
-
-    #[test]
-    fn preserves_literals_and_comments_using_lance_dialect_rules() {
-        let predicate = r#"path = '\' AND "PartyAbbrev" = 'D' -- unmatched " in comment"#;
-        assert_eq!(
-            canonicalize_sql_predicate(predicate).unwrap(),
-            r#"path = '\' AND `PartyAbbrev` = 'D' -- unmatched " in comment"#
-        );
-
-        let predicate = r#"id = 1 /* unmatched " in block comment */"#;
-        assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
-
-        let predicate = r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#;
-        assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
-    }
-
-    #[test]
-    fn rejects_unterminated_double_quoted_identifier() {
-        let error = canonicalize_sql_predicate(r#""PartyAbbrev = 'D'"#).unwrap_err();
-        assert!(matches!(error, crate::Error::InvalidInput { .. }));
+    let mut sql = run_unparser(&rewritten)?;
+    for (i, bytes) in bindings.iter().enumerate() {
+        // The unparser quotes string literals with single quotes, so the
+        // placeholder appears as `'__lancedb_binary_placeholder_<i>__'`.
+        let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i);
+        sql = sql.replace(&quoted, &bytes_to_hex_sql(bytes));
     }
+    Ok(sql)
 }

rust/lancedb/src/materialized_view.rs

@@ -170,15 +170,6 @@ pub(crate) fn plan(
     filter: Option<&str>,
     limit: Option<u64>,
 ) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
-    let filter = filter
-        .map(crate::expr::canonicalize_sql_predicate)
-        .transpose()
-        .map_err(|err| match err {
-            Error::InvalidInput { message } => Error::InvalidInput {
-                message: format!("invalid view filter: {message}"),
-            },
-            err => err,
-        })?;
     let projections: Vec<(String, String)> = if projections.is_empty() {
         source_schema
             .fields()
@@ -283,7 +274,7 @@ pub(crate) fn plan(
         declared.push(output);
     }
 
-    if let Some(filter) = filter.as_deref() {
+    if let Some(filter) = filter {
         let expr = planner
             .parse_filter(filter)
             .map_err(|e| Error::InvalidInput {
@@ -323,7 +314,7 @@ pub(crate) fn plan(
             .into_iter()
             .map(|(output, expression)| ViewProjection { output, expression })
             .collect(),
-        filter,
+        filter: filter.map(String::from),
         limit,
         inputs,
     };

rust/lancedb/src/materialized_view/refresh.rs

@@ -46,9 +46,8 @@ use lance_table::format::Fragment;
 use serde::{Deserialize, Serialize};
 
 use super::{
-    DEFINITION_META_KEY, INCARNATION_META_KEY, MaterializedViewDefinition,
-    REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
-    definition_to_metadata,
+    INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
+    SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
 };
 use crate::database::OpenTableRequest;
 use crate::table::{NativeTable, NativeTableExt, Table};
@@ -198,28 +197,8 @@ pub(crate) async fn execute_refresh(
             ),
         });
     }
-    let definition_changed =
-        definition.filter != replanned.filter || definition.inputs != replanned.inputs;
     let definition = &replanned;
 
-    // A watermark written for a legacy raw filter certifies the rows that
-    // filter produced, not the canonical predicate above. Rebuild instead of
-    // accepting or advancing it, and persist the migrated definition in the
-    // same metadata commit that certifies the replacement rows.
-    if definition_changed {
-        return rebuild(
-            view_native,
-            &view_ds,
-            &source_ds,
-            source_version,
-            source_ts,
-            definition,
-            true,
-            expected_incarnation,
-        )
-        .await;
-    }
-
     let metadata = &view_ds.schema().metadata;
     let watermark: Option<u64> = metadata
         .get(SOURCE_VERSION_META_KEY)
@@ -278,7 +257,6 @@ pub(crate) async fn execute_refresh(
                         source_version,
                         source_ts,
                         definition,
-                        false,
                         expected_incarnation,
                     )
                     .await
@@ -293,7 +271,6 @@ pub(crate) async fn execute_refresh(
                 source_version,
                 source_ts,
                 definition,
-                false,
                 expected_incarnation,
             )
             .await
@@ -706,7 +683,6 @@ async fn incremental(
             view_ds.clone(),
             source_version,
             source_ts,
-            None,
             expected_incarnation,
         )
         .await?;
@@ -728,7 +704,6 @@ async fn incremental(
             published,
             source_version,
             source_ts,
-            None,
             expected_incarnation,
         )
         .await?;
@@ -800,7 +775,6 @@ async fn incremental(
             published,
             source_version,
             source_ts,
-            None,
             expected_incarnation,
         )
         .await?;
@@ -850,14 +824,12 @@ async fn incremental(
         appended,
         source_version,
         source_ts,
-        None,
         expected_incarnation,
     )
     .await?;
     Ok(Some(result))
 }
 
-#[allow(clippy::too_many_arguments)]
 async fn rebuild(
     view_native: &NativeTable,
     view_ds: &Dataset,
@@ -865,7 +837,6 @@ async fn rebuild(
     source_version: u64,
     source_ts: u128,
     definition: &MaterializedViewDefinition,
-    persist_definition: bool,
     expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
     let rows_written = Arc::new(AtomicU64::new(0));
@@ -896,7 +867,6 @@ async fn rebuild(
         replaced,
         source_version,
         source_ts,
-        persist_definition.then_some(definition),
         expected_incarnation,
     )
     .await?;
@@ -1011,7 +981,6 @@ async fn stamp_watermark(
     mut dataset: Dataset,
     source_version: u64,
     source_ts: u128,
-    definition: Option<&MaterializedViewDefinition>,
     expected_incarnation: Option<&str>,
 ) -> Result<u64> {
     ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
@@ -1024,32 +993,27 @@ async fn stamp_watermark(
         .get(INCARNATION_META_KEY)
         .cloned()
         .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
-    let mut metadata = vec![(INCARNATION_META_KEY.to_string(), Some(incarnation))];
-    if let Some(definition) = definition {
-        metadata.push((
-            DEFINITION_META_KEY.to_string(),
-            Some(definition_to_metadata(definition)?),
-        ));
-    }
-    metadata.extend([
-        (
-            SOURCE_VERSION_META_KEY.to_string(),
-            Some(source_version.to_string()),
-        ),
-        (
-            SOURCE_VERSION_TS_META_KEY.to_string(),
-            Some(source_ts.to_string()),
-        ),
-        (
-            REFRESHED_AT_MS_META_KEY.to_string(),
-            Some(now_ms().to_string()),
-        ),
-        (
-            VIEW_VERSION_META_KEY.to_string(),
-            Some(predicted.to_string()),
-        ),
-    ]);
-    dataset.update_schema_metadata(metadata).await?;
+    dataset
+        .update_schema_metadata([
+            (INCARNATION_META_KEY.to_string(), Some(incarnation)),
+            (
+                SOURCE_VERSION_META_KEY.to_string(),
+                Some(source_version.to_string()),
+            ),
+            (
+                SOURCE_VERSION_TS_META_KEY.to_string(),
+                Some(source_ts.to_string()),
+            ),
+            (
+                REFRESHED_AT_MS_META_KEY.to_string(),
+                Some(now_ms().to_string()),
+            ),
+            (
+                VIEW_VERSION_META_KEY.to_string(),
+                Some(predicted.to_string()),
+            ),
+        ])
+        .await?;
     let actual = dataset.version().version;
     if actual != predicted {
         return Err(Error::Runtime {
@@ -1621,106 +1585,6 @@ mod tests {
         assert_eq!(read(view.table(), "x").await, vec![20, 40]);
     }
 
-    #[tokio::test]
-    async fn test_mixed_case_filter_is_canonicalized_for_lineage_and_refresh() {
-        let conn = connect("memory://").execute().await.unwrap();
-        let batch = record_batch!(
-            ("id", Int32, [1, 2, 3]),
-            ("PartyAbbrev", Utf8, ["D", "R", "D"])
-        )
-        .unwrap();
-        conn.create_table("src", batch)
-            .write_options(crate::materialized_view::tests::stable_row_ids())
-            .execute()
-            .await
-            .unwrap();
-        conn.create_materialized_view("democrats", "src")
-            .select([("id", "id")])
-            .only_if(r#""PartyAbbrev" = 'D'"#)
-            .execute()
-            .await
-            .unwrap();
-
-        // Reopen from schema metadata so these assertions cover the stored
-        // predicate and lineage, not only the declaration-time handle.
-        let view = conn.open_materialized_view("democrats").await.unwrap();
-        assert_eq!(
-            view.definition().filter.as_deref(),
-            Some("`PartyAbbrev` = 'D'")
-        );
-        assert_eq!(view.definition().inputs, ["PartyAbbrev", "id"]);
-
-        let result = view.refresh().execute().await.unwrap();
-        assert_eq!(result.rows_written, 2);
-        assert_eq!(read(view.table(), "id").await, vec![1, 3]);
-    }
-
-    #[tokio::test]
-    async fn test_legacy_raw_filter_rebuilds_and_persists_canonical_definition() {
-        let conn = connect("memory://").execute().await.unwrap();
-        let batch = record_batch!(
-            ("id", Int32, [1, 2, 3]),
-            ("PartyAbbrev", Utf8, ["D", "R", "D"])
-        )
-        .unwrap();
-        conn.create_table("legacy_src", batch)
-            .write_options(crate::materialized_view::tests::stable_row_ids())
-            .execute()
-            .await
-            .unwrap();
-        let view = conn
-            .create_materialized_view("legacy_view", "legacy_src")
-            .select([("id", "id")])
-            .only_if(r#""PartyAbbrev" = 'X'"#)
-            .execute()
-            .await
-            .unwrap();
-        assert_eq!(view.refresh().execute().await.unwrap().rows_written, 0);
-
-        // Model a definition and up-to-date watermark written before filter
-        // canonicalization was applied to materialized views.
-        let mut legacy = view.definition().clone();
-        legacy.filter = Some(r#""PartyAbbrev" = 'D'"#.into());
-        legacy.inputs = vec!["id".into()];
-        let native = view.table().as_native().unwrap();
-        let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
-        let predicted = dataset.version().version + 1;
-        dataset
-            .update_schema_metadata([
-                (
-                    DEFINITION_META_KEY.to_string(),
-                    Some(definition_to_metadata(&legacy).unwrap()),
-                ),
-                (
-                    VIEW_VERSION_META_KEY.to_string(),
-                    Some(predicted.to_string()),
-                ),
-            ])
-            .await
-            .unwrap();
-        native.dataset.update(dataset);
-
-        let reopened = conn.open_materialized_view("legacy_view").await.unwrap();
-        let result = reopened.refresh().execute().await.unwrap();
-        assert_eq!(result.mode, RefreshMode::Rebuild);
-        assert_eq!(result.rows_written, 2);
-        assert_eq!(read(reopened.table(), "id").await, vec![1, 3]);
-
-        // A fresh handle proves the migration was stored alongside the new
-        // watermark and therefore happens only once.
-        let migrated = conn.open_materialized_view("legacy_view").await.unwrap();
-        assert_eq!(
-            migrated.definition().filter.as_deref(),
-            Some("`PartyAbbrev` = 'D'")
-        );
-        assert_eq!(migrated.definition().inputs, ["PartyAbbrev", "id"]);
-        assert_eq!(
-            migrated.refresh().execute().await.unwrap().mode,
-            RefreshMode::NoOp
-        );
-        assert_eq!(read(migrated.table(), "id").await, vec![1, 3]);
-    }
-
     #[tokio::test]
     async fn test_append_refreshes_incrementally() {
         let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
@@ -2903,7 +2767,7 @@ mod tests {
         let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
         view.table().delete("x = 1").await.unwrap();
 
-        let err = stamp_watermark(view_native, stale, 99, 99, None, None).await;
+        let err = stamp_watermark(view_native, stale, 99, 99, None).await;
         assert!(err.is_err());
 
         let result = view.refresh().execute().await.unwrap();

rust/lancedb/src/query.rs

@@ -399,9 +399,6 @@ pub trait QueryBase {
     /// x > 5 OR y = 'test'
     /// ```
     ///
-    /// Identifiers may be delimited with SQL-standard double quotes or
-    /// backticks. String literals must use single quotes.
-    ///
     /// Filtering performance can often be improved by creating a scalar index
     /// on the filter column(s).
     ///
@@ -878,7 +875,7 @@ pub struct QueryRequest {
     /// [`crate::Table::set_lsm_write_spec`]) is routed through the LSM scanner so
     /// it also sees data written via the `merge_insert` LSM path that has not yet
     /// been compacted into the base table — the active and frozen in-memory
-    /// memtables and the SSTables, deduplicated by primary key
+    /// memtables and the flushed (L0) generations, deduplicated by primary key
     /// against the base table (newest generation wins); a table without a spec
     /// reads the base table.
     ///
@@ -916,17 +913,6 @@ impl QueryRequest {
     /// use different representations) the error is recorded and surfaced later
     /// by [`Self::check_filter`].
     pub(crate) fn add_filter(&mut self, new: QueryFilter) {
-        let new = match new {
-            QueryFilter::Sql(filter) => match crate::expr::canonicalize_sql_predicate(&filter) {
-                Ok(filter) => QueryFilter::Sql(filter),
-                Err(err) => {
-                    self.filter_error = Some(err.to_string());
-                    return;
-                }
-            },
-            other => other,
-        };
-
         self.filter = Some(match self.filter.take() {
             None => new,
             Some(existing) => match and_filters(existing, new) {
@@ -1188,12 +1174,12 @@ impl VectorQuery {
 
     /// Add another query vector to the search.
     ///
-    /// Multiple searches will be dispatched as a batch. Flat searches share
-    /// one table scan across the query vectors, avoiding the scan and memory
-    /// amplification of issuing the searches concurrently. Indexed searches
-    /// may still perform per-vector index work.
+    /// Multiple searches will be dispatched as part of the query.
+    /// This is a convenience method for adding multiple query vectors
+    /// to the search. It is not expected to be faster than issuing
+    /// multiple queries concurrently.
     ///
-    /// The output data will contain an additional column `query_index` which
+    /// The output data will contain an additional columns `query_index` which
     /// will contain the index of the query vector that was used to generate the
     /// result.
     pub fn add_query_vector(mut self, vector: impl IntoQueryVector) -> Result<Self> {
@@ -1660,14 +1646,10 @@ mod tests {
     use std::{collections::HashSet, sync::Arc};
 
     use super::*;
-    use arrow::{
-        array::downcast_array,
-        compute::concat_batches,
-        datatypes::{Int32Type, UInt8Type},
-    };
+    use arrow::{array::downcast_array, compute::concat_batches, datatypes::Int32Type};
     use arrow_array::{
-        FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
-        StringArray, cast::AsArray, types::Float32Type,
+        FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, cast::AsArray,
+        types::Float32Type,
     };
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
     use futures::{StreamExt, TryStreamExt};
@@ -1896,157 +1878,6 @@ mod tests {
         query.execute().await.unwrap();
     }
 
-    #[tokio::test]
-    async fn test_double_quoted_predicates_across_table_operations() {
-        let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = dataset_path.to_str().unwrap();
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new("id", DataType::Int32, false),
-            ArrowField::new("PartyAbbrev", DataType::Utf8, false),
-            ArrowField::new("path", DataType::Utf8, false),
-        ]));
-        let batch = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
-                Arc::new(StringArray::from(vec!["D", "R", "R", "D"])),
-                Arc::new(StringArray::from(vec!["\\", "\\", "x", "x"])),
-            ],
-        )
-        .unwrap();
-
-        let conn = connect(uri).execute().await.unwrap();
-        let table = conn.create_table("parties", batch).execute().await.unwrap();
-        let batches = table
-            .query()
-            .only_if(r#""PartyAbbrev" = 'D'"#)
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
-        assert_eq!(
-            table
-                .count_rows(Some(r#""PartyAbbrev" = 'D'"#.to_string()))
-                .await
-                .unwrap(),
-            2
-        );
-
-        // Public BaseTable dispatch cannot bypass canonicalization.
-        let query = AnyQuery::Query(QueryRequest {
-            filter: Some(QueryFilter::Sql(r#""PartyAbbrev" = 'D'"#.to_string())),
-            ..Default::default()
-        });
-        let batches = table
-            .base_table()
-            .query(&query, Default::default())
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 2);
-        assert_eq!(
-            table
-                .base_table()
-                .count_rows(Some(crate::table::Filter::Sql(
-                    r#""PartyAbbrev" = 'D'"#.to_string(),
-                )))
-                .await
-                .unwrap(),
-            2
-        );
-
-        for predicate in [
-            r#"id = 1 -- unmatched " in a valid SQL comment"#,
-            r#"id = 1 /* unmatched " in a valid SQL comment */"#,
-            r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#,
-            r#"path = '\' AND "PartyAbbrev" = 'D'"#,
-        ] {
-            let batches = table
-                .query()
-                .only_if(predicate)
-                .execute()
-                .await
-                .unwrap()
-                .try_collect::<Vec<_>>()
-                .await
-                .unwrap();
-            assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
-        }
-
-        // The same canonical predicate contract applies to both merge filters.
-        let source = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 2, 3])),
-                Arc::new(StringArray::from(vec!["D", "R", "R"])),
-                Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
-            ],
-        )
-        .unwrap();
-        let mut merge = table.merge_insert(&["id"]);
-        merge.when_not_matched_by_source_delete(Some(r#""PartyAbbrev" = 'D'"#.to_string()));
-        let result = table
-            .base_table()
-            .merge_insert(
-                merge,
-                Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone())),
-            )
-            .await
-            .unwrap();
-        assert_eq!(result.num_deleted_rows, 1);
-
-        let source = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 2, 3])),
-                Arc::new(StringArray::from(vec!["U", "U", "U"])),
-                Arc::new(StringArray::from(vec!["\\", "\\", "x"])),
-            ],
-        )
-        .unwrap();
-        let mut merge = table.merge_insert(&["id"]);
-        merge.when_matched_update_all(Some(r#"target."PartyAbbrev" = 'D'"#.to_string()));
-        merge
-            .execute(Box::new(RecordBatchIterator::new(vec![Ok(source)], schema)))
-            .await
-            .unwrap();
-        assert_eq!(
-            table
-                .count_rows(Some(r#""PartyAbbrev" = 'U'"#.to_string()))
-                .await
-                .unwrap(),
-            1
-        );
-
-        let update = table
-            .update()
-            .only_if(r#""PartyAbbrev" = 'R'"#)
-            .column("PartyAbbrev", "'X'");
-        table.base_table().update(update).await.unwrap();
-        assert_eq!(
-            table
-                .count_rows(Some(r#""PartyAbbrev" = 'X'"#.to_string()))
-                .await
-                .unwrap(),
-            2
-        );
-
-        let result = table
-            .base_table()
-            .delete(crate::table::Predicate::String(r#""PartyAbbrev" = 'X'"#))
-            .await
-            .unwrap();
-        assert_eq!(result.num_deleted_rows, 2);
-        assert_eq!(table.count_rows(None).await.unwrap(), 1);
-    }
-
     #[tokio::test]
     async fn test_select_with_transform() {
         let batches = make_non_empty_batches();
@@ -2503,8 +2334,7 @@ mod tests {
             .limit(1);
 
         let plan = query.explain_plan(true).await.unwrap();
-        assert!(plan.contains("KNNVectorDistance: queries=2"));
-        assert!(!plan.contains("UnionExec"));
+        assert!(plan.contains("UnionExec"));
 
         let results = query
             .execute()
@@ -2519,100 +2349,6 @@ mod tests {
         // We don't guarantee order.
         assert!(query_index.values().contains(&0));
         assert!(query_index.values().contains(&1));
-
-        // Batch KNN does not support a per-query offset, so offset queries keep
-        // the legacy per-vector plan to preserve their result semantics.
-        let offset_query = table
-            .query()
-            .nearest_to(&[0.1, 0.2, 0.3, 0.4])
-            .unwrap()
-            .add_query_vector(&[0.5, 0.6, 0.7, 0.8])
-            .unwrap()
-            .limit(1)
-            .offset(1);
-        assert!(
-            offset_query
-                .explain_plan(true)
-                .await
-                .unwrap()
-                .contains("UnionExec")
-        );
-        let offset_results = offset_query
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        assert_eq!(
-            offset_results
-                .iter()
-                .map(RecordBatch::num_rows)
-                .sum::<usize>(),
-            2
-        );
-    }
-
-    #[tokio::test]
-    async fn test_multiple_binary_query_vectors() {
-        let vectors = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
-            vec![
-                Some(vec![Some(0), Some(0)]),
-                Some(vec![Some(255), Some(255)]),
-            ],
-            2,
-        );
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new("id", DataType::Int32, false),
-            ArrowField::new("vector", vectors.data_type().clone(), false),
-        ]));
-        let batch = RecordBatch::try_new(
-            schema,
-            vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(vectors)],
-        )
-        .unwrap();
-
-        let conn = connect("memory://").execute().await.unwrap();
-        let table = conn
-            .create_table("binary_batch", batch)
-            .execute()
-            .await
-            .unwrap();
-        let query = table
-            .query()
-            .nearest_to(&[0.0, 0.0])
-            .unwrap()
-            .add_query_vector(&[255.0, 255.0])
-            .unwrap()
-            .distance_type(DistanceType::Hamming)
-            .limit(1);
-
-        // Binary queries retain the per-vector plan because Lance's binary
-        // nearest path requires primitive UInt8 query arrays.
-        assert!(
-            query
-                .explain_plan(true)
-                .await
-                .unwrap()
-                .contains("UnionExec")
-        );
-
-        let results = query
-            .execute()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        let results = concat_batches(&results[0].schema(), &results).unwrap();
-        assert_eq!(results.num_rows(), 2);
-
-        let ids = results["id"].as_primitive::<Int32Type>();
-        assert!(ids.values().contains(&0));
-        assert!(ids.values().contains(&1));
-        let query_index = results["query_index"].as_primitive::<Int32Type>();
-        assert!(query_index.values().contains(&0));
-        assert!(query_index.values().contains(&1));
     }
 
     #[tokio::test]

rust/lancedb/src/remote/table.rs

@@ -1379,11 +1379,10 @@ impl<S: HttpSend> RemoteTable<S> {
         query: &AnyQuery,
         version: Option<u64>,
     ) -> Result<Vec<serde_json::Value>> {
-        let query = query.canonicalized()?;
         let mut base_body = serde_json::json!({ "version": version });
         self.apply_branch_body(&mut base_body);
 
-        match &query {
+        match query {
             AnyQuery::Query(query) => {
                 let mut body = base_body.clone();
                 self.apply_query_params(&mut body, query)?;
@@ -2492,7 +2491,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
         let mut body = if let Some(filter) = filter {
             let filter_sql = match filter {
-                Filter::Sql(sql) => crate::expr::canonicalize_sql_predicate(&sql)?,
+                Filter::Sql(sql) => sql.clone(),
                 Filter::Datafusion(expr) => expr_to_sql_string(&expr)?,
             };
             serde_json::json!({ "predicate": filter_sql, "version": read_snapshot.version })
@@ -2748,8 +2747,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         Ok(final_analyze)
     }
 
-    async fn update(&self, mut update: UpdateBuilder) -> Result<UpdateResult> {
-        update.canonicalize_filter()?;
+    async fn update(&self, update: UpdateBuilder) -> Result<UpdateResult> {
         self.check_mutable().await?;
         let request = self
             .client
@@ -2796,7 +2794,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
     async fn delete(&self, predicate: Predicate<'_>) -> Result<DeleteResult> {
         self.check_mutable().await?;
         let predicate_sql = match predicate {
-            Predicate::String(s) => crate::expr::canonicalize_sql_predicate(s)?,
+            Predicate::String(s) => s.to_string(),
             Predicate::Expr(expr) => expr_to_sql_string(expr)?,
         };
         let mut body = serde_json::json!({ "predicate": predicate_sql });
@@ -2853,10 +2851,9 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 
     async fn merge_insert(
         &self,
-        mut params: MergeInsertBuilder,
+        params: MergeInsertBuilder,
         new_data: Box<dyn RecordBatchReader + Send>,
     ) -> Result<MergeResult> {
-        params.canonicalize_filters()?;
         self.check_mutable().await?;
 
         let timeout = params.timeout;
@@ -2951,13 +2948,13 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         Ok(())
     }
 
-    async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result<Option<LsmStats>> {
+    async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
         // Read-semantics POST, like `get_lsm_write_spec`.
         let request = self
             .client
             .post(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
             .json(&serde_json::json!({
-                "include_sstable_rows": include_sstable_rows,
+                "include_generation_rows": include_generation_rows,
             }));
         let (request_id, response) = self.send_lsm_route(request).await?;
         let body = response.text().await.err_to_http(request_id.clone())?;
@@ -3867,17 +3864,13 @@ mod tests {
             );
             assert_eq!(
                 request.body().unwrap().as_bytes().unwrap(),
-                br#"{"predicate":"`A` > 10","version":null}"#
+                br#"{"predicate":"a > 10","version":null}"#
             );
 
             http::Response::builder().status(200).body("42").unwrap()
         });
 
-        let count = table
-            .base_table()
-            .count_rows(Some(Filter::Sql(r#""A" > 10"#.into())))
-            .await
-            .unwrap();
+        let count = table.count_rows(Some("a > 10".into())).await.unwrap();
         assert_eq!(count, 42);
     }
 
@@ -4360,7 +4353,7 @@ mod tests {
                     assert_eq!(expression, "b - 1");
 
                     let only_if = value.get("predicate").unwrap().as_str().unwrap();
-                    assert_eq!(only_if, "`B` > 10");
+                    assert_eq!(only_if, "b > 10");
                 }
 
                 if old_server {
@@ -4376,12 +4369,14 @@ mod tests {
             }
         });
 
-        let update = table
+        let result = table
             .update()
             .column("a", "a + 1")
             .column("b", "b - 1")
-            .only_if(r#""B" > 10"#);
-        let result = table.base_table().update(update).await.unwrap();
+            .only_if("b > 10")
+            .execute()
+            .await
+            .unwrap();
 
         assert_eq!(result.version, if old_server { 0 } else { 43 });
         assert_eq!(result.rows_updated, if old_server { 0 } else { 5 });
@@ -4468,10 +4463,10 @@ mod tests {
 
                 let params = request.url().query_pairs().collect::<HashMap<_, _>>();
                 assert_eq!(params["on"], "some_col");
-                assert_eq!(params["when_matched_update_all"], "true");
+                assert_eq!(params["when_matched_update_all"], "false");
                 assert_eq!(params["when_not_matched_insert_all"], "false");
                 assert_eq!(params["when_not_matched_by_source_delete"], "false");
-                assert_eq!(params["when_matched_update_all_filt"], "target.`A` > 0");
+                assert!(!params.contains_key("when_matched_update_all_filt"));
                 assert!(!params.contains_key("when_not_matched_by_source_delete_filt"));
                 assert!(!params.contains_key("use_index"));
 
@@ -4488,9 +4483,11 @@ mod tests {
             }
         });
 
-        let mut merge = table.merge_insert(&["some_col"]);
-        merge.when_matched_update_all(Some(r#"target."A" > 0"#.into()));
-        let result = table.base_table().merge_insert(merge, data).await.unwrap();
+        let result = table
+            .merge_insert(&["some_col"])
+            .execute(data)
+            .await
+            .unwrap();
 
         assert_eq!(result.version, if old_server { 0 } else { 43 });
         if !old_server {
@@ -4552,7 +4549,7 @@ mod tests {
                 let body = request.body().unwrap().as_bytes().unwrap();
                 let body: serde_json::Value = serde_json::from_slice(body).unwrap();
                 let predicate = body.get("predicate").unwrap().as_str().unwrap();
-                assert_eq!(predicate, "`ID` in (1, 2, 3)");
+                assert_eq!(predicate, "id in (1, 2, 3)");
 
                 if old_server {
                     http::Response::builder()
@@ -4570,11 +4567,7 @@ mod tests {
             }
         });
 
-        let result = table
-            .base_table()
-            .delete(Predicate::String(r#""ID" in (1, 2, 3)"#))
-            .await
-            .unwrap();
+        let result = table.delete("id in (1, 2, 3)").await.unwrap();
         assert_eq!(result.version, if old_server { 0 } else { 43 });
     }
 
@@ -4666,7 +4659,6 @@ mod tests {
             let body = request.body().unwrap().as_bytes().unwrap();
             let body: serde_json::Value = serde_json::from_slice(body).unwrap();
             let expected_body = serde_json::json!({
-                "filter": "`A` > 0",
                 "k": isize::MAX as usize,
                 "prefilter": true,
                 "vector": [], // Empty vector means no vector query.
@@ -4682,13 +4674,9 @@ mod tests {
                 .unwrap()
         });
 
-        let query = AnyQuery::Query(QueryRequest {
-            filter: Some(QueryFilter::Sql(r#""A" > 0"#.into())),
-            ..Default::default()
-        });
         let data = table
-            .base_table()
-            .query(&query, Default::default())
+            .query()
+            .execute()
             .await
             .unwrap()
             .collect::<Vec<_>>()
@@ -8260,7 +8248,7 @@ mod tests {
         http::Response::builder().status(200).body(body).unwrap()
     }
 
-    /// A flush landing in an empty SSTable tier finishes on the opening stats read
+    /// A flush landing in an empty L0 finishes on the opening stats read
     /// alone. Asserting zero compacts is the point: "it returned Ok" is also
     /// true of a loop that ran a pointless pass.
     #[tokio::test(start_paused = true)]
@@ -8314,7 +8302,7 @@ mod tests {
     }
 
     /// Generations created *during* the checkpoint are not waited on, which
-    /// is what lets the loop terminate on a table taking writes where "the SSTable tier is
+    /// is what lets the loop terminate on a table taking writes where "L0 is
     /// empty" never becomes true.
     #[tokio::test(start_paused = true)]
     async fn test_checkpoint_ignores_generations_created_while_it_runs() {
@@ -8593,7 +8581,7 @@ mod tests {
     }
 
     /// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
-    /// defaulting to a zero it did not measure. `include_sstable_rows`
+    /// defaulting to a zero it did not measure. `include_generation_rows`
     /// rides in the body and is off unless asked for.
     #[tokio::test]
     async fn test_get_lsm_stats_round_trip() {
@@ -8602,7 +8590,7 @@ mod tests {
             let body = request.body().unwrap().as_bytes().unwrap();
             let body: serde_json::Value = serde_json::from_slice(body).unwrap();
             assert_eq!(
-                body["include_sstable_rows"], true,
+                body["include_generation_rows"], true,
                 "the flag must reach the server, not be silently dropped"
             );
             let response = serde_json::json!({

rust/lancedb/src/table.rs

@@ -102,7 +102,7 @@ use futures::future::join_all;
 pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags};
 pub use lance::dataset::scanner::DatasetRecordBatchStream;
 pub use lance_index::optimize::OptimizeOptions;
-pub use lsm_stats::{LsmStats, MemtableStats, SsTableStats, TableShardStats};
+pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats};
 pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats};
 pub use refresh::RefreshColumnResult;
 pub use schema_evolution::{
@@ -673,7 +673,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
             message: "get_lsm_write_spec is not supported on this table type".into(),
         })
     }
-    /// Freeze every table shard's active memtable into an SSTable.
+    /// Seal every bucket's active memtable into L0.
     ///
     /// The default implementation returns `NotSupported`.
     async fn flush_lsm(&self) -> Result<()> {
@@ -681,7 +681,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
             message: "flush_lsm is not supported on this table type".into(),
         })
     }
-    /// Trigger a background SSTable compaction pass per table shard.
+    /// Trigger a background L0 → base compaction pass per bucket.
     ///
     /// The default implementation returns `NotSupported`.
     async fn compact_lsm(&self) -> Result<()> {
@@ -693,7 +693,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     /// enabled for this table.
     ///
     /// The default implementation returns `NotSupported`.
-    async fn get_lsm_stats(&self, _include_sstable_rows: bool) -> Result<Option<LsmStats>> {
+    async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result<Option<LsmStats>> {
         Err(Error::NotSupported {
             message: "get_lsm_stats is not supported on this table type".into(),
         })
@@ -1164,10 +1164,7 @@ impl Table {
     ///
     /// * `filter` if present, only count rows matching the filter
     pub async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
-        let filter = filter
-            .map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate).map(Filter::Sql))
-            .transpose()?;
-        self.inner.count_rows(filter).await
+        self.inner.count_rows(filter.map(Filter::Sql)).await
     }
 
     /// Names of the blob v2 columns in this table, in declaration order.
@@ -1367,13 +1364,7 @@ impl Table {
     /// # });
     /// ```
     pub async fn delete(&self, predicate: impl Into<Predicate<'_>>) -> Result<DeleteResult> {
-        match predicate.into() {
-            Predicate::String(predicate) => {
-                let predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
-                self.inner.delete(Predicate::String(&predicate)).await
-            }
-            predicate @ Predicate::Expr(_) => self.inner.delete(predicate).await,
-        }
+        self.inner.delete(predicate.into()).await
     }
 
     /// Create an index on the provided column(s).
@@ -1786,23 +1777,7 @@ impl Table {
         self.inner.alter_columns(alterations).await
     }
 
-    /// Update per-field (column) metadata.
-    ///
-    /// Each [`FieldMetadataUpdate`] is merged into the field's existing metadata
-    /// by default; use [`FieldMetadataUpdate::remove`] to delete a key, or
-    /// [`FieldMetadataUpdate::replace`] to swap the field's entire metadata map.
-    ///
-    /// The following keys are treated specially, by convention, and should be
-    /// used when appropriate:
-    ///
-    /// - `lancedb:description`: for a human-readable description of a field.
-    /// - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
-    ///   names the tag category; e.g. `lancedb:tag:model: "clip"`.
-    /// - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
-    ///   `feature_v2` might be in the same logical column.
-    /// - `lancedb:status`: for status options (`production`, `candidate`,
-    ///   `deprecated`, `archived`) to designate the current life cycle state of
-    ///   this column.
+    /// Update per-field metadata (merges by default).
     pub async fn update_field_metadata(
         &self,
         updates: &[FieldMetadataUpdate],
@@ -1897,7 +1872,7 @@ impl Table {
 
     /// Converge this table's LSM write path into its base table.
     ///
-    /// One `flush` to freeze every memtable into an SSTable, then compaction triggers
+    /// One `flush` to seal every memtable into L0, then compaction triggers
     /// until every generation that existed at that moment has reached base.
     /// The loop runs client-side, reading progress from `get_lsm_stats`, so
     /// there is no held socket and nothing to reconcile if you drop this
@@ -1932,10 +1907,10 @@ impl Table {
         checkpoint::checkpoint_lsm(self).await
     }
 
-    /// Freeze every table shard's active memtable into an SSTable without touching the
+    /// Seal every bucket's active memtable into L0 without touching the
     /// base table.
     ///
-    /// Independently useful: flushing makes memtable rows readable from an SSTable at
+    /// Independently useful: flushing makes memtable rows readable from L0 at
     /// a lower per-query cost. On a node that has not claimed this table it
     /// claims it and replays the WAL log first — reporting "nothing to flush"
     /// without replaying would lie about durable data.
@@ -1943,7 +1918,7 @@ impl Table {
         self.inner.flush_lsm().await
     }
 
-    /// Run one bounded SSTable compaction pass per table shard, reporting what
+    /// Run one bounded L0 → base compaction pass per bucket, reporting what
     /// it merged and what is left.
     ///
     /// One pass, not convergence: that bounds each request's cost and gives a
@@ -1959,7 +1934,7 @@ impl Table {
     /// state, though on a node that has not claimed this table it claims it,
     /// exactly as a read would.
     ///
-    /// `include_sstable_rows` reports a row count per SSTable. Off by
+    /// `include_generation_rows` reports a row count per L0 generation. Off by
     /// default: each count opens an uncached Lance dataset, and
     /// `checkpoint_lsm` polls this needing only generation numbers.
     ///
@@ -1970,8 +1945,8 @@ impl Table {
     ///
     /// Do not build a checkpoint's termination on this: the completion
     /// predicate lives in the `flush` and `compact` responses.
-    pub async fn get_lsm_stats(&self, include_sstable_rows: bool) -> Result<Option<LsmStats>> {
-        self.inner.get_lsm_stats(include_sstable_rows).await
+    pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
+        self.inner.get_lsm_stats(include_generation_rows).await
     }
 
     /// Drain and close any cached MemWAL shard writers held for this table.
@@ -3248,10 +3223,7 @@ impl BaseTable for NativeTable {
         let dataset = self.dataset.get().await?;
         match filter {
             None => Ok(dataset.count_rows(None).await?),
-            Some(Filter::Sql(sql)) => {
-                let sql = crate::expr::canonicalize_sql_predicate(&sql)?;
-                Ok(dataset.count_rows(Some(sql)).await?)
-            }
+            Some(Filter::Sql(sql)) => Ok(dataset.count_rows(Some(sql)).await?),
             Some(Filter::Datafusion(_)) => Err(Error::NotSupported {
                 message: "Datafusion filters are not yet supported".to_string(),
             }),

rust/lancedb/src/table/checkpoint.rs

@@ -4,7 +4,7 @@
 //! Converging a table's LSM write path into its base table.
 //!
 //! `checkpoint_lsm` seals once, then triggers compaction and watches
-//! generation numbers until the SSTables that existed at the start are gone.
+//! generation numbers until the L0 that existed at the start is gone.
 //!
 //! The loop runs in the client, not the server: `compact_lsm` dispatches a
 //! pass and returns, so nothing holds a socket and a client can vanish
@@ -150,7 +150,7 @@ where
 }
 
 /// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
-/// from the resulting SSTables, then trigger and poll until they drain.
+/// from the resulting L0, then trigger and poll until it drains.
 pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
     for reissue in 0..=MAX_REISSUES {
         // The seal turns everything written before this call into a
@@ -177,9 +177,9 @@ pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
             return Ok(());
         };
         let targets: HashMap<String, u64> = stats
-            .table_shards
+            .buckets
             .iter()
-            .filter_map(|b| Some((b.shard_id.clone(), b.newest_sstable_generation()?)))
+            .filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?)))
             .collect();
         if targets.is_empty() {
             return Ok(());
@@ -226,11 +226,11 @@ async fn drain_to_targets(
         // with nothing outstanding are skipped, not counted as idle.
         let mut outstanding = 0;
         let mut all_compacting = true;
-        for b in &stats.table_shards {
+        for b in &stats.buckets {
             let Some(target) = targets.get(&b.shard_id) else {
                 continue;
             };
-            let n = b.outstanding_sstables(*target);
+            let n = b.outstanding_generations(*target);
             if n > 0 {
                 outstanding += n;
                 all_compacting &= b.compacting;

rust/lancedb/src/table/create_index.rs

@@ -133,7 +133,7 @@ impl NativeTable {
                     ),
                 });
             }
-            (resolved.canonical_path, resolved.field)
+            (resolved.canonical_path, resolved.terminal_field)
         } else {
             Self::resolve_index_field(dataset.schema(), &opts.columns[0])?
         };
@@ -439,7 +439,8 @@ mod tests {
     use arrow_array::record_batch;
     use arrow_array::{
         Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeListArray, Float32Array, Int32Array,
-        LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StructArray,
+        LargeBinaryArray, LargeStringArray, ListArray, RecordBatch, StringArray, StructArray,
+        UInt32Array,
     };
     use arrow_data::ArrayDataBuilder;
     use arrow_schema::{DataType, Field, Schema};
@@ -458,6 +459,7 @@ mod tests {
     use crate::query::{ExecutableQuery, QueryBase};
     use crate::table::optimize::{CompactionOptions, OptimizeAction};
     use lance_index::scalar::FullTextSearchQuery;
+    use lance_index::scalar::inverted::query::{FtsQuery, MatchQuery};
 
     fn create_fixed_size_list<T: Array>(
         values: T,
@@ -599,6 +601,80 @@ mod tests {
         assert!(invalid_granularity.is_err());
     }
 
+    #[tokio::test]
+    async fn test_nested_list_fts_uses_deepest_document_coordinates() {
+        let conn = connect("memory://").execute().await.unwrap();
+        let mut docs = ListBuilder::new(ListBuilder::new(StringBuilder::new()));
+
+        docs.values().values().append_value("alpha");
+        docs.values().values().append_value("beta");
+        docs.values().append(true);
+        docs.values().values().append_value("gamma");
+        docs.values().values().append_value("alpha delta");
+        docs.values().append(true);
+        docs.append(true);
+
+        docs.values().append(true);
+        docs.values().values().append_value("alpha");
+        docs.values().append(true);
+        docs.append(true);
+
+        let batch = RecordBatch::try_from_iter(vec![
+            ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef),
+            ("docs", Arc::new(docs.finish()) as ArrayRef),
+        ])
+        .unwrap();
+        let table = conn.create_table("nested", batch).execute().await.unwrap();
+
+        let job = table
+            .create_index(
+                &["docs"],
+                Index::FTS(
+                    FtsIndexBuilder::default()
+                        .document_granularity(DocumentGranularity::ListElement),
+                ),
+            )
+            .execute_async()
+            .await
+            .unwrap();
+        job.wait().await.unwrap();
+
+        let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+            MatchQuery::new("alpha".to_string())
+                .with_column(Some("docs".to_string()))
+                .with_document_granularity(DocumentGranularity::ListElement),
+        ));
+        let batches = table
+            .query()
+            .full_text_search(query)
+            .limit(10)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let mut hits = Vec::new();
+        for batch in batches {
+            let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
+            let coordinates = batch["_doc_index"]
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap();
+            for row in 0..batch.num_rows() {
+                let coordinate = coordinates.value(row);
+                let coordinate = coordinate.as_any().downcast_ref::<UInt32Array>().unwrap();
+                hits.push((ids.value(row), coordinate.values().to_vec()));
+            }
+        }
+        hits.sort_unstable();
+        assert_eq!(
+            hits,
+            vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]
+        );
+    }
+
     /// Concurrent waiters, and a wait issued after the job settled, all
     /// succeed once the build does.
     #[tokio::test]

rust/lancedb/src/table/delete.rs

@@ -31,9 +31,8 @@ pub(crate) async fn execute_delete(
     table.dataset.ensure_mutable()?;
     match predicate {
         Predicate::String(s) => {
-            let predicate = crate::expr::canonicalize_sql_predicate(s)?;
             let mut dataset = (*table.dataset.get().await?).clone();
-            let delete_result = dataset.delete(&predicate).boxed().await?;
+            let delete_result = dataset.delete(s).boxed().await?;
             let num_deleted_rows = delete_result.num_deleted_rows;
             let version = dataset.version().version;
             table.dataset.update(dataset);

rust/lancedb/src/table/lsm_stats.rs

@@ -1,21 +1,21 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-//! Live per-table_shard LSM state — the shape [`crate::Table::get_lsm_stats`]
+//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`]
 //! returns and [`super::checkpoint`] polls.
 //!
-//! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag)
+//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag)
 //! are the caller's to compute. There is no "WAL is off" shape — that case is
 //! `None`, because a struct of zeros would read as measurements.
 
 use serde::Deserialize;
 
-/// One SSTable.
+/// One flushed L0 generation.
 #[derive(Debug, Clone, Deserialize)]
-pub struct SsTableStats {
+pub struct GenerationStats {
     pub generation: u64,
     pub bytes: u64,
-    /// Present only when `include_sstable_rows` was requested. Off by
+    /// Present only when `include_generation_rows` was requested. Off by
     /// default because each count opens an uncached Lance dataset, and the
     /// checkpoint loop polls this route needing only generation numbers.
     #[serde(default)]
@@ -34,11 +34,11 @@ pub struct MemtableStats {
     pub indexes: Vec<String>,
 }
 
-/// Live state of one table_shard. A table is N table_shards on one node; flattening to
-/// a single number hides the one hot table_shard that is usually why someone
+/// Live state of one bucket. A table is N buckets on one node; flattening to
+/// a single number hides the one hot bucket that is usually why someone
 /// opened this endpoint.
 #[derive(Debug, Clone, Deserialize)]
-pub struct TableShardStats {
+pub struct BucketStats {
     pub shard_id: String,
     /// `Active` | `Sealed` (drop-table 2PC in flight).
     pub status: String,
@@ -47,42 +47,42 @@ pub struct TableShardStats {
     pub current_generation: u64,
     pub replay_after_wal_entry_position: u64,
     pub wal_entry_position_last_seen: u64,
-    pub sstables: Vec<SsTableStats>,
-    /// Whether a pass owns this table_shard's compaction latch right now. Says *a*
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
     /// driver is running, not *whose*, and the latch is held from dispatch —
     /// including while the pass queues for a pod-wide compactor permit. Read
     /// it as "do not pile on", never as "mine is progressing".
     pub compacting: bool,
-    /// Oldest first, active last. Absent for a `Sealed` table_shard, whose
+    /// Oldest first, active last. Absent for a `Sealed` bucket, whose
     /// in-memory state is torn down.
     #[serde(default)]
     pub memtables: Option<Vec<MemtableStats>>,
 }
 
-impl TableShardStats {
-    /// The newest SSTable generation, or `None` when the tier is empty.
-    pub(crate) fn newest_sstable_generation(&self) -> Option<u64> {
-        self.sstables.iter().map(|g| g.generation).max()
+impl BucketStats {
+    /// The newest flushed generation, or `None` when L0 is empty.
+    pub(crate) fn newest_generation(&self) -> Option<u64> {
+        self.generations.iter().map(|g| g.generation).max()
     }
 
-    /// How many SSTables at or below `target` are still uncompacted.
+    /// How many generations at or below `target` are still in L0.
     ///
     /// A count, not a boolean: one pass drains a bounded prefix rather than
     /// the whole target set, so a boolean would read as "no progress" for
     /// every pass but the last. Compaction drains oldest-first, so this
     /// decreases monotonically.
-    pub(crate) fn outstanding_sstables(&self, target: u64) -> usize {
-        self.sstables
+    pub(crate) fn outstanding_generations(&self, target: u64) -> usize {
+        self.generations
             .iter()
             .filter(|g| g.generation <= target)
             .count()
     }
 }
 
-/// Live LSM state, one entry per table_shard.
+/// Live LSM state, one entry per bucket.
 #[derive(Debug, Clone, Deserialize)]
 pub struct LsmStats {
-    pub table_shards: Vec<TableShardStats>,
+    pub buckets: Vec<BucketStats>,
 }
 
 /// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
@@ -97,18 +97,18 @@ pub(crate) struct GetLsmStatsResponse {
 mod tests {
     use super::*;
 
-    fn table_shard(shard: &str, sstables: &[u64], compacting: bool) -> TableShardStats {
-        TableShardStats {
+    fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats {
+        BucketStats {
             shard_id: shard.into(),
             status: "Active".into(),
             writer_epoch: 1,
             manifest_version: 1,
-            current_generation: sstables.iter().max().copied().unwrap_or(0) + 1,
+            current_generation: generations.iter().max().copied().unwrap_or(0) + 1,
             replay_after_wal_entry_position: 0,
             wal_entry_position_last_seen: 0,
-            sstables: sstables
+            generations: generations
                 .iter()
-                .map(|g| SsTableStats {
+                .map(|g| GenerationStats {
                     generation: *g,
                     bytes: 1,
                     rows: None,
@@ -123,46 +123,40 @@ mod tests {
     /// generation created after it must not hold the loop open — that is why
     /// the predicate terminates under write load.
     #[test]
-    fn newer_sstables_do_not_extend_the_target() {
-        let start = table_shard("b0", &[7, 8], false);
-        let target = start
-            .newest_sstable_generation()
-            .expect("the SSTable tier is non-empty");
+    fn newer_generations_do_not_extend_the_target() {
+        let start = bucket("b0", &[7, 8], false);
+        let target = start.newest_generation().expect("L0 is non-empty");
         assert_eq!(target, 8);
 
         // Compaction drained 7 and 8; 9 and 10 arrived while it ran.
-        let later = table_shard("b0", &[9, 10], false);
+        let later = bucket("b0", &[9, 10], false);
         assert_eq!(
-            later.outstanding_sstables(target),
+            later.outstanding_generations(target),
             0,
-            "sstables above the target are somebody else's problem"
+            "generations above the target are somebody else's problem"
         );
 
         // Still holding 8 means still outstanding.
         assert_eq!(
-            table_shard("b0", &[8, 9], false).outstanding_sstables(target),
+            bucket("b0", &[8, 9], false).outstanding_generations(target),
             1
         );
     }
 
-    /// The metric counts SSTables, not table shards: a pass drains a bounded
-    /// prefix, so one table_shard going 3 → 2 → 1 → 0 is three steps.
+    /// The metric counts generations, not buckets: a pass drains a bounded
+    /// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps.
     #[test]
-    fn progress_is_measured_in_sstables() {
+    fn progress_is_measured_in_generations() {
         let target = 3;
         let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
             .iter()
-            .map(|gens| table_shard("b0", gens, false).outstanding_sstables(target))
+            .map(|gens| bucket("b0", gens, false).outstanding_generations(target))
             .collect();
         assert_eq!(counts, vec![3, 2, 1, 0]);
     }
 
     #[test]
-    fn an_empty_sstable_tier_has_no_target() {
-        assert!(
-            table_shard("b0", &[], false)
-                .newest_sstable_generation()
-                .is_none()
-        );
+    fn empty_l0_has_no_target() {
+        assert!(bucket("b0", &[], false).newest_generation().is_none());
     }
 }

rust/lancedb/src/table/merge.rs

@@ -220,32 +220,9 @@ impl MergeInsertBuilder {
     ///
     /// Returns version and statistics about the merge operation including the number of rows
     /// inserted, updated, and deleted.
-    pub async fn execute(
-        mut self,
-        new_data: Box<dyn RecordBatchReader + Send>,
-    ) -> Result<MergeResult> {
-        self.canonicalize_filters()?;
+    pub async fn execute(self, new_data: Box<dyn RecordBatchReader + Send>) -> Result<MergeResult> {
         self.table.clone().merge_insert(self, new_data).await
     }
-
-    pub(crate) fn canonicalize_filters(&mut self) -> Result<()> {
-        self.when_matched_update_all_filt =
-            canonicalize_merge_filter(self.when_matched_update_all_filt.take())?;
-        self.when_not_matched_by_source_delete_filt =
-            canonicalize_merge_filter(self.when_not_matched_by_source_delete_filt.take())?;
-        Ok(())
-    }
-}
-
-fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<MergeFilter>> {
-    filter
-        .map(|filter| match filter {
-            MergeFilter::Sql(predicate) => {
-                crate::expr::canonicalize_sql_predicate(&predicate).map(MergeFilter::Sql)
-            }
-            filter @ MergeFilter::Expr(_) => Ok(filter),
-        })
-        .transpose()
 }
 
 /// Internal implementation of the merge insert logic
@@ -253,10 +230,9 @@ fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<Merge
 /// This logic was moved from NativeTable::merge_insert to keep table.rs clean.
 pub(crate) async fn execute_merge_insert(
     table: &NativeTable,
-    mut params: MergeInsertBuilder,
+    params: MergeInsertBuilder,
     new_data: Box<dyn RecordBatchReader + Send>,
 ) -> Result<MergeResult> {
-    params.canonicalize_filters()?;
     super::computed_columns::ensure_no_function_bindings_for_mutation(
         table.schema().await?.as_ref(),
         "merge_insert",

rust/lancedb/src/table/query.rs

@@ -17,11 +17,11 @@ use arrow::array::{AsArray, FixedSizeListBuilder, Float32Builder};
 use arrow::datatypes::{Float32Type, UInt8Type};
 use arrow_array::Array;
 use arrow_schema::{DataType, Schema};
-use datafusion_common::{Column, DataFusionError, SchemaError};
 use datafusion_physical_plan::ExecutionPlan;
 use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::repartition::RepartitionExec;
 use datafusion_physical_plan::union::UnionExec;
+use futures::future::try_join_all;
 use lance::dataset::mem_wal::DatasetMemWalExt;
 use lance::dataset::scanner::DatasetRecordBatchStream;
 use lance::dataset::scanner::Scanner;
@@ -45,22 +45,6 @@ impl AnyQuery {
             Self::VectorQuery(query) => &query.base,
         }
     }
-
-    fn base_mut(&mut self) -> &mut QueryRequest {
-        match self {
-            Self::Query(query) => query,
-            Self::VectorQuery(query) => &mut query.base,
-        }
-    }
-
-    /// Canonicalize any raw SQL filter immediately before backend dispatch.
-    pub(crate) fn canonicalized(&self) -> Result<Self> {
-        let mut query = self.clone();
-        if let Some(QueryFilter::Sql(predicate)) = &mut query.base_mut().filter {
-            *predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
-        }
-        Ok(query)
-    }
 }
 
 //Decide between namespace or local
@@ -69,16 +53,15 @@ pub async fn execute_query(
     query: &AnyQuery,
     options: QueryExecutionOptions,
 ) -> Result<DatasetRecordBatchStream> {
-    let query = query.canonicalized()?;
     // QueryTable pushdown runs the query server-side, but only on the main
     // branch: the namespace request carries no branch yet, so a branch handle
     // must fall through to local execution.
-    if can_execute_namespace_query(table, &query).await?
+    if can_execute_namespace_query(table, query).await?
         && let Some(ref namespace_client) = table.namespace_client
     {
-        return execute_namespace_query(table, namespace_client.clone(), &query, options).await;
+        return execute_namespace_query(table, namespace_client.clone(), query, options).await;
     }
-    execute_generic_query(table, &query, options).await
+    execute_generic_query(table, query, options).await
 }
 
 async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> Result<bool> {
@@ -153,10 +136,9 @@ pub async fn create_plan(
     query: &AnyQuery,
     options: QueryExecutionOptions,
 ) -> Result<Arc<dyn ExecutionPlan>> {
-    let query = query.canonicalized()?;
     let query = match query {
-        AnyQuery::VectorQuery(query) => query,
-        AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query),
+        AnyQuery::VectorQuery(query) => query.clone(),
+        AnyQuery::Query(query) => VectorQueryRequest::from_plain_query(query.clone()),
     };
     query.base.check_filter()?;
 
@@ -188,48 +170,26 @@ pub async fn create_plan(
     let mut column = query.column.clone();
 
     let mut query_vector = query.query_vector.first().cloned();
-    let mut is_batch_query = false;
     if query.query_vector.len() > 1 {
         if column.is_none() {
             // Infer a vector column with the same dimension of the query vector.
-            let arrow_schema = Schema::from(schema);
+            let arrow_schema = Schema::from(ds_ref.schema());
             column = Some(default_vector_column(
                 &arrow_schema,
                 Some(query.query_vector[0].len() as i32),
             )?);
         }
         let vector_field = schema.field(column.as_ref().unwrap()).unwrap();
-        let (_, element_type) =
-            lance::index::vector::utils::get_vector_type(schema, column.as_ref().unwrap())?;
-        let is_binary = matches!(element_type, DataType::UInt8);
-        if matches!(vector_field.data_type(), DataType::List(_))
-            || (query.base.offset.unwrap_or(0) == 0 && !is_binary)
-        {
-            // Lance distinguishes these cases from the vector column type: a
-            // list-like query against a List column is one multivector query,
-            // while the same query against a FixedSizeList column is a batch of
-            // independent queries. The batch path shares a single flat scan and
-            // bounds retained candidate data instead of running one scan per
-            // query vector.
+        if let DataType::List(_) = vector_field.data_type() {
+            // Multivector handling: concatenate into FixedSizeList<FixedSizeList<_>>
             let vectors = query
                 .query_vector
                 .iter()
                 .map(|arr| arr.as_ref())
                 .collect::<Vec<_>>();
             let dim = vectors[0].len();
-            if let Some((query_index, actual_dim)) = vectors
-                .iter()
-                .enumerate()
-                .find_map(|(index, vector)| (vector.len() != dim).then_some((index, vector.len())))
-            {
-                return Err(Error::InvalidInput {
-                    message: format!(
-                        "query vector at index {query_index} has dimension {actual_dim}, expected {dim}"
-                    ),
-                });
-            }
             let mut fsl_builder = FixedSizeListBuilder::with_capacity(
-                Float32Builder::with_capacity(dim * vectors.len()),
+                Float32Builder::with_capacity(dim),
                 dim as i32,
                 vectors.len(),
             );
@@ -240,12 +200,8 @@ pub async fn create_plan(
                 fsl_builder.append(true);
             }
             query_vector = Some(Arc::new(fsl_builder.finish()));
-            is_batch_query = !matches!(vector_field.data_type(), DataType::List(_));
         } else {
-            // Lance's batch path has no per-query offset, and its binary path
-            // requires primitive UInt8 queries rather than a fixed-size list.
-            // Keep the prior plan shape for these cases so offsets are applied
-            // per query and binary query vectors retain their primitive shape.
+            // Multiple query vectors: create a plan for each and union them
             let query_vecs = query.query_vector.clone();
             let plan_futures = query_vecs
                 .into_iter()
@@ -258,7 +214,7 @@ pub async fn create_plan(
                     }
                 })
                 .collect::<Vec<_>>();
-            let plans = futures::future::try_join_all(plan_futures).await?;
+            let plans = try_join_all(plan_futures).await?;
             return create_multi_vector_plan(plans);
         }
     }
@@ -269,7 +225,7 @@ pub async fn create_plan(
         let column = if let Some(col) = column {
             col
         } else {
-            let arrow_schema = Schema::from(schema);
+            let arrow_schema = Schema::from(ds_ref.schema());
             default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
         };
 
@@ -295,14 +251,10 @@ pub async fn create_plan(
         }
     }
 
-    // For a batch query, `nearest` already applies k to each query vector.
-    // Adding Scanner's global limit would truncate the combined result to k rows.
-    if !is_batch_query {
-        scanner.limit(
-            query.base.limit.map(|limit| limit as i64),
-            query.base.offset.map(|offset| offset as i64),
-        )?;
-    }
+    scanner.limit(
+        query.base.limit.map(|limit| limit as i64),
+        query.base.offset.map(|offset| offset as i64),
+    )?;
 
     if let Some(ef) = query.ef {
         scanner.ef(ef);
@@ -375,97 +327,7 @@ pub async fn create_plan(
         scanner.order_by(Some(order_by.clone()))?;
     }
 
-    scanner
-        .create_plan()
-        .await
-        .map_err(|error| enrich_lance_field_not_found(error, schema))
-}
-
-/// Replace DataFusion's top-level field candidates with qualified leaf paths.
-///
-/// DataFusion resolves nested fields but its `FieldNotFound` error only lists the
-/// top-level Arrow fields. This makes a missing leaf look unavailable even when it
-/// exists below a struct. Keep every other Lance/DataFusion error unchanged and
-/// enrich only this one schema error at the LanceDB query boundary.
-fn enrich_lance_field_not_found(
-    error: lance::Error,
-    schema: &lance_core::datatypes::Schema,
-) -> Error {
-    let Some(field) = find_missing_field(&error) else {
-        return error.into();
-    };
-    field_not_found_error(field, &Schema::from(schema))
-}
-
-fn field_not_found_diagnostic(
-    error: &(dyn std::error::Error + 'static),
-    schema: &Schema,
-) -> Option<Error> {
-    let field = find_missing_field(error)?;
-    Some(field_not_found_error(field, schema))
-}
-
-fn field_not_found_error(field: &Column, schema: &Schema) -> Error {
-    let valid_fields = leaf_field_paths(schema);
-    let mut message = format!("Schema error: No field named {}", field.quoted_flat_name());
-    if !valid_fields.is_empty() {
-        message.push_str(". Valid fields are ");
-        message.push_str(&valid_fields.join(", "));
-    }
-    message.push('.');
-
-    Error::InvalidInput { message }
-}
-
-fn find_missing_field<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a Column> {
-    if let Some(DataFusionError::SchemaError(schema_error, _)) =
-        error.downcast_ref::<DataFusionError>()
-        && let SchemaError::FieldNotFound { field, .. } = schema_error.as_ref()
-    {
-        return Some(field);
-    }
-
-    error.source().and_then(find_missing_field)
-}
-
-fn leaf_field_paths(schema: &Schema) -> Vec<String> {
-    fn format_segment(segment: &str) -> String {
-        // Quote every segment instead of maintaining a SQL keyword list. Bare
-        // lowercase names such as `true` can be parsed as expressions rather
-        // than identifiers, while backticks preserve all field names in both
-        // local SQL parsers.
-        format!("`{}`", segment.replace('`', "``"))
-    }
-
-    fn visit(fields: &arrow_schema::Fields, path: &mut Vec<String>, paths: &mut Vec<String>) {
-        for field in fields {
-            // Neither local planner can address an empty field-path segment,
-            // even when it is backtick-quoted. Do not advertise leaves beneath
-            // such a segment as valid filter fields.
-            if field.name().is_empty() {
-                continue;
-            }
-            path.push(field.name().clone());
-            match field.data_type() {
-                DataType::Struct(children) if !children.is_empty() => {
-                    visit(children, path, paths);
-                }
-                _ => {
-                    paths.push(
-                        path.iter()
-                            .map(|segment| format_segment(segment))
-                            .collect::<Vec<_>>()
-                            .join("."),
-                    );
-                }
-            }
-            path.pop();
-        }
-    }
-
-    let mut paths = Vec::new();
-    visit(schema.fields(), &mut Vec::new(), &mut paths);
-    paths
+    Ok(scanner.create_plan().await?)
 }
 
 //Helper functions below
@@ -825,10 +687,7 @@ async fn parse_arrow_ipc_response(bytes: bytes::Bytes) -> Result<DatasetRecordBa
 #[cfg(test)]
 #[allow(deprecated)]
 mod tests {
-    use arrow_array::{
-        ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
-        StructArray,
-    };
+    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array};
     use futures::TryStreamExt;
     use lance_arrow::FixedSizeListArrayExt;
     use std::sync::{
@@ -837,7 +696,7 @@ mod tests {
     };
 
     use super::*;
-    use crate::query::{ExecutableQuery, QueryBase, QueryExecutionOptions, QueryRequest};
+    use crate::query::{QueryExecutionOptions, QueryRequest};
     use crate::table::BaseTable;
 
     fn fixed_size_list_array(values: Vec<f32>, dimension: i32) -> FixedSizeListArray {
@@ -978,6 +837,7 @@ mod tests {
     async fn test_execute_query_local_routing() {
         use crate::connect;
         use crate::table::query::execute_query;
+        use arrow_array::{Int32Array, RecordBatch};
         use arrow_schema::{DataType, Field, Schema};
 
         let conn = connect("memory://").execute().await.unwrap();
@@ -1017,164 +877,6 @@ mod tests {
         assert_eq!(count, 2); // 4 and 5
     }
 
-    #[tokio::test]
-    async fn test_missing_filter_field_lists_nested_fields_in_local_planners() {
-        use crate::connect;
-        use arrow_schema::{DataType, Field, Schema};
-
-        let conn = connect("memory://").execute().await.unwrap();
-        let metadata = Arc::new(StructArray::from(vec![
-            (
-                Arc::new(Field::new("year", DataType::Int32, false)),
-                Arc::new(Int32Array::from(vec![2024])) as ArrayRef,
-            ),
-            (
-                Arc::new(Field::new("genre", DataType::Utf8, false)),
-                Arc::new(StringArray::from(vec!["fiction"])) as ArrayRef,
-            ),
-            (
-                Arc::new(Field::new("Title", DataType::Int32, false)),
-                Arc::new(Int32Array::from(vec![7])) as ArrayRef,
-            ),
-            (
-                Arc::new(Field::new("true", DataType::Int32, false)),
-                Arc::new(Int32Array::from(vec![8])) as ArrayRef,
-            ),
-            (
-                Arc::new(Field::new("", DataType::Int32, false)),
-                Arc::new(Int32Array::from(vec![10])) as ArrayRef,
-            ),
-        ]));
-        let vector = Arc::new(fixed_size_list_array(vec![0.0, 1.0], 2));
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::Int32, false),
-            Field::new("vector", vector.data_type().clone(), false),
-            Field::new("content", DataType::Utf8, false),
-            Field::new("metadata", metadata.data_type().clone(), false),
-        ]));
-        let batch = RecordBatch::try_new(
-            schema,
-            vec![
-                Arc::new(Int32Array::from(vec![1])),
-                vector,
-                Arc::new(StringArray::from(vec!["example"])),
-                metadata,
-            ],
-        )
-        .unwrap();
-        let table = conn
-            .create_table("nested_error", batch)
-            .execute()
-            .await
-            .unwrap();
-
-        let error = table
-            .query()
-            .only_if("year = 2024")
-            .execute()
-            .await
-            .err()
-            .expect("query should reject the unqualified nested field");
-        let case_sensitive_path = "`metadata`.`Title`";
-        let keyword_path = "`metadata`.`true`";
-        let expected = format!(
-            "No field named year. Valid fields are `id`, `vector`, `content`, `metadata`.`year`, `metadata`.`genre`, {case_sensitive_path}, {keyword_path}."
-        );
-
-        assert!(
-            error.to_string().contains(&expected),
-            "unexpected error: {error}"
-        );
-        for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
-            table
-                .query()
-                .only_if(format!("{path} = {value}"))
-                .execute()
-                .await
-                .expect("the path advertised by the diagnostic should be reusable");
-        }
-
-        table.set_unenforced_primary_key(["id"]).await.unwrap();
-        table
-            .set_lsm_write_spec(crate::table::LsmWriteSpec::unsharded())
-            .await
-            .unwrap();
-        let lsm_error = table
-            .query()
-            .only_if("year = 2024")
-            .execute()
-            .await
-            .err()
-            .expect("LSM query should reject the unqualified nested field");
-
-        assert!(
-            lsm_error.to_string().contains(&expected),
-            "unexpected LSM error: {lsm_error}"
-        );
-        for (path, value) in [(case_sensitive_path, 7), (keyword_path, 8)] {
-            table
-                .query()
-                .only_if(format!("{path} = {value}"))
-                .execute()
-                .await
-                .expect("the path advertised by the diagnostic should be reusable in LSM queries");
-        }
-    }
-
-    #[test]
-    fn test_leaf_field_paths_preserve_arbitrary_depth() {
-        use arrow_schema::{DataType, Field, Schema};
-
-        fn nested_field(path: &[&str]) -> Field {
-            let mut segments = path.iter().rev();
-            let mut field = Field::new(
-                *segments.next().expect("path must have a leaf"),
-                DataType::Int32,
-                false,
-            );
-            for segment in segments {
-                field = Field::new(*segment, DataType::Struct(vec![field].into()), false);
-            }
-            field
-        }
-
-        let schema = Schema::new(vec![
-            nested_field(&["a", "b", "c", "d", "e"]),
-            nested_field(&["metadata", "child.with.dot"]),
-            nested_field(&["metadata", "Title"]),
-            nested_field(&["metadata", "123child"]),
-            nested_field(&["metadata", "child`tick"]),
-            nested_field(&["metadata", ""]),
-            nested_field(&["", "child"]),
-        ]);
-
-        assert_eq!(
-            leaf_field_paths(&schema),
-            vec![
-                "`a`.`b`.`c`.`d`.`e`",
-                "`metadata`.`child.with.dot`",
-                "`metadata`.`Title`",
-                "`metadata`.`123child`",
-                "`metadata`.`child``tick`",
-            ]
-        );
-
-        let source = DataFusionError::SchemaError(
-            Box::new(SchemaError::FieldNotFound {
-                field: Box::new(Column::from_name("missing")),
-                valid_fields: Vec::new(),
-            }),
-            Box::new(None),
-        );
-        let error = field_not_found_diagnostic(&source, &schema).unwrap();
-        assert!(
-            error.to_string().contains(
-                "Valid fields are `a`.`b`.`c`.`d`.`e`, `metadata`.`child.with.dot`, `metadata`.`Title`, `metadata`.`123child`, `metadata`.`child``tick`"
-            ),
-            "unexpected error: {error}"
-        );
-    }
-
     #[derive(Debug, Default)]
     struct CountingNamespaceClient {
         query_table_calls: AtomicUsize,
@@ -1386,7 +1088,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_create_plan_batch_vector_uses_shared_scan() {
+    async fn test_create_plan_multivector_structure() {
         use arrow_array::{Float32Array, RecordBatch};
         use arrow_schema::{DataType, Field, Schema};
         use datafusion_physical_plan::display::DisplayableExecutionPlan;
@@ -1413,18 +1115,11 @@ mod tests {
             .unwrap();
         let native_table = table.as_native().unwrap();
 
-        // A batch of vectors against a fixed-size vector column should use
-        // Lance's native batch KNN path instead of independent scan plans.
+        // This triggers the "create_multi_vector_plan" logic branch
         let q1 = Arc::new(Float32Array::from(vec![1.0, 2.0]));
         let q2 = Arc::new(Float32Array::from(vec![3.0, 4.0]));
 
         let req = VectorQueryRequest {
-            base: QueryRequest {
-                filter: Some(QueryFilter::Sql("id >= 0".to_string())),
-                limit: Some(1),
-                select: Select::Columns(vec!["id".to_string()]),
-                ..Default::default()
-            },
             column: Some("vector".to_string()),
             query_vector: vec![q1, q2],
             ..Default::default()
@@ -1441,17 +1136,19 @@ mod tests {
             .indent(true)
             .to_string();
 
+        // We expect a RepartitionExec wrapping a UnionExec
         assert!(
-            display.contains("KNNVectorDistance: queries=2"),
-            "plan should use native batch KNN, got:\n{display}"
+            display.contains("RepartitionExec"),
+            "Plan should include Repartitioning"
         );
         assert!(
-            !display.contains("UnionExec"),
-            "flat batch KNN should share one scan, got:\n{display}"
+            display.contains("UnionExec"),
+            "Plan should include a Union of multiple searches"
         );
+        // We expect the projection to add the 'query_index' column (logic inside multi_vector_plan)
         assert!(
             display.contains("query_index"),
-            "plan should add query_index column, got:\n{display}"
+            "Plan should add query_index column"
         );
     }
 

rust/lancedb/src/table/query/lsm.rs

@@ -27,8 +27,6 @@ use std::sync::Arc;
 
 use arrow_array::Array;
 use arrow_schema::{DataType, Schema as ArrowSchema};
-use datafusion::common::{DataFusionError, ToDFSchema};
-use datafusion::prelude::SessionContext;
 use datafusion_physical_plan::expressions::Column;
 use datafusion_physical_plan::projection::ProjectionExec;
 use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
@@ -393,21 +391,7 @@ fn base_scanner(
     }
     if let Some(filter) = &query.base.filter {
         scanner = match filter {
-            QueryFilter::Sql(sql) => {
-                // Parse here instead of inside `LsmScanner::filter` so the typed
-                // DataFusion `FieldNotFound` error is still available for the
-                // same nested-field enrichment used by the ordinary scanner.
-                let schema = ArrowSchema::from(dataset.schema());
-                let df_schema = schema.clone().to_dfschema().map_err(|error| {
-                    enrich_filter_error(error, &schema, "Failed to create DFSchema")
-                })?;
-                let expr = SessionContext::new()
-                    .parse_sql_expr(sql, &df_schema)
-                    .map_err(|error| {
-                        enrich_filter_error(error, &schema, "Failed to parse filter expression")
-                    })?;
-                scanner.filter_expr(expr)
-            }
+            QueryFilter::Sql(sql) => scanner.filter(sql)?,
             QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()),
             QueryFilter::Substrait(_) => {
                 return Err(Error::NotSupported {
@@ -419,12 +403,6 @@ fn base_scanner(
     Ok(scanner)
 }
 
-fn enrich_filter_error(error: DataFusionError, schema: &ArrowSchema, context: &str) -> Error {
-    super::field_not_found_diagnostic(&error, schema).unwrap_or_else(|| Error::InvalidInput {
-        message: format!("{context}: {error}"),
-    })
-}
-
 /// Plain scan: filter / projection / limit over base ∪ SSTables ∪ in-memory.
 /// The plain scan applies limit and offset inside the planner.
 async fn plain_plan(

rust/lancedb/src/table/schema_evolution.rs

@@ -55,9 +55,7 @@ pub struct DropColumnsResult {
 pub struct FieldMetadataUpdate {
     /// Dot-separated path to the field (e.g. `"embedding"` or `"address.zip"`).
     pub path: String,
-    /// Keys to set (`Some`) or delete (`None`). See
-    /// [`Table::update_field_metadata`](crate::Table::update_field_metadata) for
-    /// the conventional `lancedb:*` keys.
+    /// Keys to set (`Some`) or delete (`None`).
     pub metadata: HashMap<String, Option<String>>,
     /// If `true`, replace the field's entire metadata map instead of merging.
     pub replace: bool,

rust/lancedb/src/table/update.rs

@@ -62,33 +62,22 @@ impl UpdateBuilder {
     }
 
     /// Executes the update operation.
-    pub async fn execute(mut self) -> Result<UpdateResult> {
+    pub async fn execute(self) -> Result<UpdateResult> {
         if self.columns.is_empty() {
             Err(Error::InvalidInput {
                 message: "at least one column must be specified in an update operation".to_string(),
             })
         } else {
-            self.canonicalize_filter()?;
             self.parent.clone().update(self).await
         }
     }
-
-    pub(crate) fn canonicalize_filter(&mut self) -> Result<()> {
-        self.filter = self
-            .filter
-            .take()
-            .map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate))
-            .transpose()?;
-        Ok(())
-    }
 }
 
 /// Internal implementation of the update logic
 pub(crate) async fn execute_update(
     table: &NativeTable,
-    mut update: UpdateBuilder,
+    update: UpdateBuilder,
 ) -> Result<UpdateResult> {
-    update.canonicalize_filter()?;
     table.dataset.ensure_mutable()?;
 
     // 1. Snapshot the current dataset

rust/lancedb/src/utils/mod.rs

@@ -227,7 +227,7 @@ pub(crate) fn resolve_arrow_field_path(schema: &Schema, column: &str) -> Result<
 
 pub(crate) struct ResolvedFtsField {
     pub canonical_path: String,
-    pub field: Field,
+    pub terminal_field: Field,
     pub list_depth: usize,
 }
 
@@ -309,7 +309,7 @@ pub(crate) fn resolve_lance_fts_field_path(
     );
     Ok(ResolvedFtsField {
         canonical_path,
-        field: Field::from(field),
+        terminal_field: Field::from(terminal),
         list_depth,
     })
 }
@@ -375,7 +375,7 @@ pub(crate) fn resolve_arrow_fts_field_path(
             message: format!("Invalid schema: {}", e),
         })?;
     let resolved = resolve_lance_fts_field_path(&lance_schema, column)?;
-    Ok((resolved.canonical_path, resolved.field))
+    Ok((resolved.canonical_path, resolved.terminal_field))
 }
 
 pub fn supported_btree_data_type(dtype: &DataType) -> bool {
@@ -647,8 +647,9 @@ mod tests {
             Field::new("docs", text_list(), true),
         ]);
 
-        let (path, _) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
+        let (path, field) = resolve_arrow_fts_field_path(&schema, "docs.content").unwrap();
         assert_eq!(path, "docs.content");
+        assert_eq!(field.data_type(), &DataType::Utf8);
 
         let lance_schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
         let field_id = lance_schema