fix small details

Closes #2191


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Chores**
- Updated the required version of the pyarrow package to version 16 or
higher.
- Adjusted automated testing workflows to install pyarrow version 16 for
compatibility checks.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.2-beta.1"
+current_version = "0.19.1-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/docs.yml

@@ -18,17 +18,24 @@ concurrency:
   group: "pages"
   cancel-in-progress: true
 
+env:
+  # This reduces the disk space needed for the build
+  RUSTFLAGS: "-C debuginfo=0"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+
 jobs:
   # Single deploy job since we're just deploying
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependecies needed for ubuntu
+      - name: Install dependencies needed for ubuntu
         run: |
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update && rustup default
@@ -38,6 +45,7 @@ jobs:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: "docs/requirements.txt"
+      - uses: Swatinem/rust-cache@v2
       - name: Build Python
         working-directory: python
         run: |
@@ -49,7 +57,6 @@ jobs:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
       - name: Install node dependencies
         working-directory: node
         run: |

.github/workflows/java-publish.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: "1.79.0"
+          toolchain: "1.81.0"
           cache-workspaces: "./java/core/lancedb-jni"
           # Disable full debug symbol generation to speed up CI build and keep memory down
           # "1" means line tables only, which is useful for panic tracebacks.
@@ -97,7 +97,7 @@ jobs:
       - name: Dry run
         if: github.event_name == 'pull_request'
         run: |
-          mvn --batch-mode -DskipTests package
+          mvn --batch-mode -DskipTests -Drust.release.build=true package
       - name: Set github
         run: |
           git config --global user.email "LanceDB Github Runner"
@@ -108,7 +108,7 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
+          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}

.github/workflows/npm-publish.yml

@@ -18,6 +18,7 @@ on:
     # This should trigger a dry run (we skip the final publish step)
     paths:
       - .github/workflows/npm-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -130,29 +131,24 @@ jobs:
               set -e &&
               apt-get update &&
               apt-get install -y protobuf-compiler pkg-config
-
-          # TODO: re-enable x64 musl builds. I could not figure out why, but it
-          # consistently made GHA runners non-responsive at the end of build. Example:
-          # https://github.com/lancedb/lancedb/actions/runs/13980431071/job/39144319470?pr=2250
-
-          # - target: x86_64-unknown-linux-musl
-          #   # This one seems to need some extra memory
-          #   host: ubuntu-2404-8x-x64
-          #   # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
-          #   docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
-          #   features: ","
-          #   pre_build: |-
-          #     set -e &&
-          #     apk add protobuf-dev curl &&
-          #     ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
-          #     ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so
-
+          - target: x86_64-unknown-linux-musl
+            # This one seems to need some extra memory
+            host: ubuntu-2404-8x-x64
+            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
+            features: fp16kernels
+            pre_build: |-
+              set -e &&
+              apk add protobuf-dev curl &&
+              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
+              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
+              CC=gcc &&
+              CXX=g++
           - target: aarch64-unknown-linux-gnu
             host: ubuntu-2404-8x-x64
             # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
-            # TODO: enable fp16kernels after https://github.com/lancedb/lance/pull/3559
-            features: ","
+            features: "fp16kernels"
             pre_build: |-
               set -e &&
               apt-get update &&
@@ -170,8 +166,8 @@ jobs:
               set -e &&
               apk add protobuf-dev &&
               rustup target add aarch64-unknown-linux-musl &&
-              export CC="/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc" &&
-              export CXX="/aarch64-linux-musl-cross/bin/aarch64-linux-musl-g++"
+              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
+              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
     name: build - ${{ matrix.settings.target }}
     runs-on: ${{ matrix.settings.host }}
     defaults:
@@ -535,6 +531,12 @@ jobs:
           for filename in *.tgz; do
             npm publish $PUBLISH_ARGS $filename
           done
+      - name: Deprecate
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        # We need to deprecate the old package to avoid confusion.
+        # Each time we publish a new version, it gets undeprecated.
+        run: npm deprecate vectordb "Use @lancedb/lancedb instead."
       - name: Notify Slack Action
         uses: ravsamhq/notify-slack-action@2.3.0
         if: ${{ always() }}

.github/workflows/pypi-publish.yml

@@ -8,6 +8,7 @@ on:
     # This should trigger a dry run (we skip the final publish step)
     paths:
       - .github/workflows/pypi-publish.yml
+      - Cargo.toml # Change in dependency frequently breaks builds
 
 jobs:
   linux:

.github/workflows/python.yml

@@ -136,9 +136,9 @@ jobs:
       - uses: ./.github/workflows/run_tests
         with:
           integration: true
-      - name: Test without pylance
+      - name: Test without pylance or pandas
         run: |
-          pip uninstall -y pylance
+          pip uninstall -y pylance pandas
           pytest -vv python/tests/test_table.py
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
@@ -228,6 +228,7 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
+          pip install pyarrow==16
           pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests

Cargo.toml

@@ -21,16 +21,14 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.25.0", "features" = [
-    "dynamodb",
-] }
-lance-io = { version = "=0.25.0" }
-lance-index = { version = "=0.25.0" }
-lance-linalg = { version = "=0.25.0" }
-lance-table = { version = "=0.25.0" }
-lance-testing = { version = "=0.25.0" }
-lance-datafusion = { version = "=0.25.0" }
-lance-encoding = { version = "=0.25.0" }
+lance = { "version" = "=0.27.0", "features" = ["dynamodb"], tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-io = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-index = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-linalg = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-table = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-testing = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-datafusion = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
+lance-encoding = { version = "=0.27.0", tag = "v0.27.0-beta.2", git="https://github.com/lancedb/lance.git" }
 # Note that this one does not include pyarrow
 arrow = { version = "54.1", optional = false }
 arrow-array = "54.1"
@@ -41,12 +39,12 @@ arrow-schema = "54.1"
 arrow-arith = "54.1"
 arrow-cast = "54.1"
 async-trait = "0"
-datafusion = { version = "45.0", default-features = false }
-datafusion-catalog = "45.0"
-datafusion-common = { version = "45.0", default-features = false }
-datafusion-execution = "45.0"
-datafusion-expr = "45.0"
-datafusion-physical-plan = "45.0"
+datafusion = { version = "46.0", default-features = false }
+datafusion-catalog = "46.0"
+datafusion-common = { version = "46.0", default-features = false }
+datafusion-execution = "46.0"
+datafusion-expr = "46.0"
+datafusion-physical-plan = "46.0"
 env_logger = "0.11"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",

docs/README.md

@@ -2,7 +2,7 @@
 
 LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
 
-Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+Docs is built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
 whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
 unreleased features.
 

docs/mkdocs.yml

@@ -105,7 +105,8 @@ markdown_extensions:
 nav:
   - Home:
       - LanceDB: index.md
-      - 🏃🏼‍♂️ Quick start: basic.md
+      - 👉 Quickstart: quickstart.md
+      - 🏃🏼‍♂️ Basic Usage: basic.md
       - 📚 Concepts:
           - Vector search: concepts/vector_search.md
           - Indexing:
@@ -237,7 +238,9 @@ nav:
           - 👾 JavaScript (lancedb): js/globals.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
 
-  - Quick start: basic.md
+  - Getting Started:
+      - Quickstart: quickstart.md
+      - Basic Usage: basic.md
   - Concepts:
       - Vector search: concepts/vector_search.md
       - Indexing:

docs/src/basic.md

@@ -1,4 +1,4 @@
-# Quick start
+# Basic Usage
 
 !!! info "LanceDB can be run in a number of ways:"
 

docs/src/guides/storage.md

@@ -342,7 +342,7 @@ For **read and write access**, LanceDB will need a policy such as:
             "Action": [
               "s3:PutObject",
               "s3:GetObject",
-              "s3:DeleteObject",
+              "s3:DeleteObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },
@@ -374,7 +374,7 @@ For **read-only access**, LanceDB will need a policy such as:
         {
             "Effect": "Allow",
             "Action": [
-              "s3:GetObject",
+              "s3:GetObject"
             ],
             "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
         },

docs/src/guides/tables.md

@@ -765,7 +765,10 @@ This can be used to update zero to all rows depending on how many rows match the
         ];
         const tbl = await db.createTable("my_table", data)
 
-        await tbl.update({vector: [10, 10]}, { where: "x = 2"})
+        await tbl.update({ 
+            values: { vector: [10, 10] },
+            where: "x = 2"
+        });
         ```
 
     === "vectordb (deprecated)"
@@ -784,7 +787,10 @@ This can be used to update zero to all rows depending on how many rows match the
         ];
         const tbl = await db.createTable("my_table", data)
 
-        await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        await tbl.update({ 
+            where: "x = 2", 
+            values: { vector: [10, 10] } 
+        });
         ```
 
 #### Updating using a sql query

docs/src/js/classes/BoostQuery.md

@@ -0,0 +1,67 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / BoostQuery
+
+# Class: BoostQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new BoostQuery()
+
+```ts
+new BoostQuery(
+   positive,
+   negative,
+   options?): BoostQuery
+```
+
+Creates an instance of BoostQuery.
+The boost returns documents that match the positive query,
+but penalizes those that match the negative query.
+the penalty is controlled by the `negativeBoost` parameter.
+
+#### Parameters
+
+* **positive**: [`FullTextQuery`](../interfaces/FullTextQuery.md)
+    The positive query that boosts the relevance score.
+
+* **negative**: [`FullTextQuery`](../interfaces/FullTextQuery.md)
+    The negative query that reduces the relevance score.
+
+* **options?**
+    Optional parameters for the boost query.
+    - `negativeBoost`: The boost factor for the negative query (default is 0.0).
+
+* **options.negativeBoost?**: `number`
+
+#### Returns
+
+[`BoostQuery`](BoostQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/MatchQuery.md

@@ -0,0 +1,70 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MatchQuery
+
+# Class: MatchQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new MatchQuery()
+
+```ts
+new MatchQuery(
+   query,
+   column,
+   options?): MatchQuery
+```
+
+Creates an instance of MatchQuery.
+
+#### Parameters
+
+* **query**: `string`
+    The text query to search for.
+
+* **column**: `string`
+    The name of the column to search within.
+
+* **options?**
+    Optional parameters for the match query.
+    - `boost`: The boost factor for the query (default is 1.0).
+    - `fuzziness`: The fuzziness level for the query (default is 0).
+    - `maxExpansions`: The maximum number of terms to consider for fuzzy matching (default is 50).
+
+* **options.boost?**: `number`
+
+* **options.fuzziness?**: `number`
+
+* **options.maxExpansions?**: `number`
+
+#### Returns
+
+[`MatchQuery`](MatchQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/MergeInsertBuilder.md

@@ -33,20 +33,20 @@ Construct a MergeInsertBuilder. __Internal use only.__
 ### execute()
 
 ```ts
-execute(data): Promise<void>
+execute(data): Promise<MergeStats>
 ```
 
 Executes the merge insert operation
 
-Nothing is returned but the `Table` is updated
-
 #### Parameters
 
 * **data**: [`Data`](../type-aliases/Data.md)
 
 #### Returns
 
-`Promise`&lt;`void`&gt;
+`Promise`&lt;[`MergeStats`](../interfaces/MergeStats.md)&gt;
+
+Statistics about the merge operation: counts of inserted, updated, and deleted rows
 
 ***
 

docs/src/js/classes/MultiMatchQuery.md

@@ -0,0 +1,64 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MultiMatchQuery
+
+# Class: MultiMatchQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new MultiMatchQuery()
+
+```ts
+new MultiMatchQuery(
+   query,
+   columns,
+   options?): MultiMatchQuery
+```
+
+Creates an instance of MultiMatchQuery.
+
+#### Parameters
+
+* **query**: `string`
+    The text query to search for across multiple columns.
+
+* **columns**: `string`[]
+    An array of column names to search within.
+
+* **options?**
+    Optional parameters for the multi-match query.
+    - `boosts`: An array of boost factors for each column (default is 1.0 for all).
+
+* **options.boosts?**: `number`[]
+
+#### Returns
+
+[`MultiMatchQuery`](MultiMatchQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/PhraseQuery.md

@@ -0,0 +1,55 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / PhraseQuery
+
+# Class: PhraseQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Implements
+
+- [`FullTextQuery`](../interfaces/FullTextQuery.md)
+
+## Constructors
+
+### new PhraseQuery()
+
+```ts
+new PhraseQuery(query, column): PhraseQuery
+```
+
+Creates an instance of `PhraseQuery`.
+
+#### Parameters
+
+* **query**: `string`
+    The phrase to search for in the specified column.
+
+* **column**: `string`
+    The name of the column to search within.
+
+#### Returns
+
+[`PhraseQuery`](PhraseQuery.md)
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)
+
+#### Implementation of
+
+[`FullTextQuery`](../interfaces/FullTextQuery.md).[`queryType`](../interfaces/FullTextQuery.md#querytype)

docs/src/js/classes/Query.md

@@ -30,6 +30,53 @@ protected inner: Query | Promise<Query>;
 
 ## Methods
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+
+***
+
 ### execute()
 
 ```ts
@@ -159,7 +206,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 
@@ -262,7 +309,7 @@ nearestToText(query, columns?): Query
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **columns?**: `string`[]
 

docs/src/js/classes/QueryBase.md

@@ -36,6 +36,49 @@ protected inner: NativeQueryType | Promise<NativeQueryType>;
 
 ## Methods
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+***
+
 ### execute()
 
 ```ts
@@ -149,7 +192,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 

docs/src/js/classes/Table.md

@@ -117,8 +117,8 @@ wish to return to standard mode, call `checkoutLatest`.
 
 #### Parameters
 
-* **version**: `number`
-    The version to checkout
+* **version**: `string` \| `number`
+    The version to checkout, could be version number or tag
 
 #### Returns
 
@@ -454,6 +454,28 @@ Modeled after ``VACUUM`` in PostgreSQL.
 
 ***
 
+### prewarmIndex()
+
+```ts
+abstract prewarmIndex(name): Promise<void>
+```
+
+Prewarm an index in the table.
+
+#### Parameters
+
+* **name**: `string`
+    The name of the index.
+    This will load the index into memory.  This may reduce the cold-start time for
+    future queries.  If the index does not fit in the cache then this call may be
+    wasteful.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
 ### query()
 
 ```ts
@@ -575,7 +597,7 @@ of the given query
 
 #### Parameters
 
-* **query**: `string` \| [`IntoVector`](../type-aliases/IntoVector.md)
+* **query**: `string` \| [`IntoVector`](../type-aliases/IntoVector.md) \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
     the query, a vector or string
 
 * **queryType?**: `string`
@@ -593,6 +615,50 @@ of the given query
 
 ***
 
+### stats()
+
+```ts
+abstract stats(): Promise<TableStatistics>
+```
+
+Returns table and fragment statistics
+
+#### Returns
+
+`Promise`&lt;[`TableStatistics`](../interfaces/TableStatistics.md)&gt;
+
+The table and fragment statistics
+
+***
+
+### tags()
+
+```ts
+abstract tags(): Promise<Tags>
+```
+
+Get a tags manager for this table.
+
+Tags allow you to label specific versions of a table with a human-readable name.
+The returned tags manager can be used to list, create, update, or delete tags.
+
+#### Returns
+
+`Promise`&lt;[`Tags`](Tags.md)&gt;
+
+A tags manager for this table
+
+#### Example
+
+```typescript
+const tagsManager = await table.tags();
+await tagsManager.create("v1", 1);
+const tags = await tagsManager.list();
+console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
+```
+
+***
+
 ### toArrow()
 
 ```ts
@@ -731,3 +797,26 @@ Retrieve the version of the table
 #### Returns
 
 `Promise`&lt;`number`&gt;
+
+***
+
+### waitForIndex()
+
+```ts
+abstract waitForIndex(indexNames, timeoutSeconds): Promise<void>
+```
+
+Waits for asynchronous indexing to complete on the table.
+
+#### Parameters
+
+* **indexNames**: `string`[]
+    The name of the indices to wait for
+
+* **timeoutSeconds**: `number`
+    The number of seconds to wait before timing out
+    This will raise an error if the indices are not created and fully indexed within the timeout.
+
+#### Returns
+
+`Promise`&lt;`void`&gt;

docs/src/js/classes/TagContents.md

@@ -0,0 +1,35 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TagContents
+
+# Class: TagContents
+
+## Constructors
+
+### new TagContents()
+
+```ts
+new TagContents(): TagContents
+```
+
+#### Returns
+
+[`TagContents`](TagContents.md)
+
+## Properties
+
+### manifestSize
+
+```ts
+manifestSize: number;
+```
+
+***
+
+### version
+
+```ts
+version: number;
+```

docs/src/js/classes/Tags.md

@@ -0,0 +1,99 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Tags
+
+# Class: Tags
+
+## Constructors
+
+### new Tags()
+
+```ts
+new Tags(): Tags
+```
+
+#### Returns
+
+[`Tags`](Tags.md)
+
+## Methods
+
+### create()
+
+```ts
+create(tag, version): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+* **version**: `number`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
+### delete()
+
+```ts
+delete(tag): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;
+
+***
+
+### getVersion()
+
+```ts
+getVersion(tag): Promise<number>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+#### Returns
+
+`Promise`&lt;`number`&gt;
+
+***
+
+### list()
+
+```ts
+list(): Promise<Record<string, TagContents>>
+```
+
+#### Returns
+
+`Promise`&lt;`Record`&lt;`string`, [`TagContents`](TagContents.md)&gt;&gt;
+
+***
+
+### update()
+
+```ts
+update(tag, version): Promise<void>
+```
+
+#### Parameters
+
+* **tag**: `string`
+
+* **version**: `number`
+
+#### Returns
+
+`Promise`&lt;`void`&gt;

docs/src/js/classes/VectorQuery.md

@@ -48,6 +48,53 @@ addQueryVector(vector): VectorQuery
 
 ***
 
+### analyzePlan()
+
+```ts
+analyzePlan(): Promise<string>
+```
+
+Executes the query and returns the physical query plan annotated with runtime metrics.
+
+This is useful for debugging and performance analysis, as it shows how the query was executed
+and includes metrics such as elapsed time, rows processed, and I/O statistics.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A query execution plan with runtime metrics for each step.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+
+const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+
+Example output (with runtime metrics inlined):
+AnalyzeExec verbose=true, metrics=[]
+ ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+  Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+    GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+     FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+      SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+       KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+        LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`analyzePlan`](QueryBase.md#analyzeplan)
+
+***
+
 ### bypassVectorIndex()
 
 ```ts
@@ -300,7 +347,7 @@ fullTextSearch(query, options?): this
 
 #### Parameters
 
-* **query**: `string`
+* **query**: `string` \| [`FullTextQuery`](../interfaces/FullTextQuery.md)
 
 * **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
 

docs/src/js/enumerations/FullTextQueryType.md

@@ -0,0 +1,46 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FullTextQueryType
+
+# Enumeration: FullTextQueryType
+
+Enum representing the types of full-text queries supported.
+
+- `Match`: Performs a full-text search for terms in the query string.
+- `MatchPhrase`: Searches for an exact phrase match in the text.
+- `Boost`: Boosts the relevance score of specific terms in the query.
+- `MultiMatch`: Searches across multiple fields for the query terms.
+
+## Enumeration Members
+
+### Boost
+
+```ts
+Boost: "boost";
+```
+
+***
+
+### Match
+
+```ts
+Match: "match";
+```
+
+***
+
+### MatchPhrase
+
+```ts
+MatchPhrase: "match_phrase";
+```
+
+***
+
+### MultiMatch
+
+```ts
+MultiMatch: "multi_match";
+```

docs/src/js/globals.md

@@ -9,16 +9,26 @@
 - [embedding](namespaces/embedding/README.md)
 - [rerankers](namespaces/rerankers/README.md)
 
+## Enumerations
+
+- [FullTextQueryType](enumerations/FullTextQueryType.md)
+
 ## Classes
 
+- [BoostQuery](classes/BoostQuery.md)
 - [Connection](classes/Connection.md)
 - [Index](classes/Index.md)
 - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
+- [MatchQuery](classes/MatchQuery.md)
 - [MergeInsertBuilder](classes/MergeInsertBuilder.md)
+- [MultiMatchQuery](classes/MultiMatchQuery.md)
+- [PhraseQuery](classes/PhraseQuery.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
 - [RecordBatchIterator](classes/RecordBatchIterator.md)
 - [Table](classes/Table.md)
+- [TagContents](classes/TagContents.md)
+- [Tags](classes/Tags.md)
 - [VectorColumnOptions](classes/VectorColumnOptions.md)
 - [VectorQuery](classes/VectorQuery.md)
 
@@ -32,7 +42,10 @@
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
+- [FragmentStatistics](interfaces/FragmentStatistics.md)
+- [FragmentSummaryStats](interfaces/FragmentSummaryStats.md)
 - [FtsOptions](interfaces/FtsOptions.md)
+- [FullTextQuery](interfaces/FullTextQuery.md)
 - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
 - [HnswPqOptions](interfaces/HnswPqOptions.md)
 - [HnswSqOptions](interfaces/HnswSqOptions.md)
@@ -41,6 +54,7 @@
 - [IndexStatistics](interfaces/IndexStatistics.md)
 - [IvfFlatOptions](interfaces/IvfFlatOptions.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
+- [MergeStats](interfaces/MergeStats.md)
 - [OpenTableOptions](interfaces/OpenTableOptions.md)
 - [OptimizeOptions](interfaces/OptimizeOptions.md)
 - [OptimizeStats](interfaces/OptimizeStats.md)
@@ -48,6 +62,7 @@
 - [RemovalStats](interfaces/RemovalStats.md)
 - [RetryConfig](interfaces/RetryConfig.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
+- [TableStatistics](interfaces/TableStatistics.md)
 - [TimeoutConfig](interfaces/TimeoutConfig.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
 - [Version](interfaces/Version.md)

docs/src/js/interfaces/FragmentStatistics.md

@@ -0,0 +1,37 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FragmentStatistics
+
+# Interface: FragmentStatistics
+
+## Properties
+
+### lengths
+
+```ts
+lengths: FragmentSummaryStats;
+```
+
+Statistics on the number of rows in the table fragments
+
+***
+
+### numFragments
+
+```ts
+numFragments: number;
+```
+
+The number of fragments in the table
+
+***
+
+### numSmallFragments
+
+```ts
+numSmallFragments: number;
+```
+
+The number of uncompacted fragments in the table

docs/src/js/interfaces/FragmentSummaryStats.md

@@ -0,0 +1,77 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FragmentSummaryStats
+
+# Interface: FragmentSummaryStats
+
+## Properties
+
+### max
+
+```ts
+max: number;
+```
+
+The number of rows in the fragment with the most rows
+
+***
+
+### mean
+
+```ts
+mean: number;
+```
+
+The mean number of rows in the fragments
+
+***
+
+### min
+
+```ts
+min: number;
+```
+
+The number of rows in the fragment with the fewest rows
+
+***
+
+### p25
+
+```ts
+p25: number;
+```
+
+The 25th percentile of number of rows in the fragments
+
+***
+
+### p50
+
+```ts
+p50: number;
+```
+
+The 50th percentile of number of rows in the fragments
+
+***
+
+### p75
+
+```ts
+p75: number;
+```
+
+The 75th percentile of number of rows in the fragments
+
+***
+
+### p99
+
+```ts
+p99: number;
+```
+
+The 99th percentile of number of rows in the fragments

docs/src/js/interfaces/FullTextQuery.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FullTextQuery
+
+# Interface: FullTextQuery
+
+Represents a full-text query interface.
+This interface defines the structure and behavior for full-text queries,
+including methods to retrieve the query type and convert the query to a dictionary format.
+
+## Methods
+
+### queryType()
+
+```ts
+queryType(): FullTextQueryType
+```
+
+The type of the full-text query.
+
+#### Returns
+
+[`FullTextQueryType`](../enumerations/FullTextQueryType.md)

docs/src/js/interfaces/IndexOptions.md

@@ -39,3 +39,11 @@ and the same name, then an error will be returned.  This is true even if
 that index is out of date.
 
 The default is true
+
+***
+
+### waitTimeoutSeconds?
+
+```ts
+optional waitTimeoutSeconds: number;
+```

docs/src/js/interfaces/MergeStats.md

@@ -0,0 +1,31 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MergeStats
+
+# Interface: MergeStats
+
+## Properties
+
+### numDeletedRows
+
+```ts
+numDeletedRows: bigint;
+```
+
+***
+
+### numInsertedRows
+
+```ts
+numInsertedRows: bigint;
+```
+
+***
+
+### numUpdatedRows
+
+```ts
+numUpdatedRows: bigint;
+```

docs/src/js/interfaces/QueryExecutionOptions.md

@@ -20,3 +20,13 @@ The maximum number of rows to return in a single batch
 
 Batches may have fewer rows if the underlying data is stored
 in smaller chunks.
+
+***
+
+### timeoutMs?
+
+```ts
+optional timeoutMs: number;
+```
+
+Timeout for query execution in milliseconds

docs/src/js/interfaces/TableStatistics.md

@@ -0,0 +1,47 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TableStatistics
+
+# Interface: TableStatistics
+
+## Properties
+
+### fragmentStats
+
+```ts
+fragmentStats: FragmentStatistics;
+```
+
+Statistics on table fragments
+
+***
+
+### numIndices
+
+```ts
+numIndices: number;
+```
+
+The number of indices in the table
+
+***
+
+### numRows
+
+```ts
+numRows: number;
+```
+
+The number of rows in the table
+
+***
+
+### totalBytes
+
+```ts
+totalBytes: number;
+```
+
+The total number of bytes in the table

docs/src/quickstart.md

@@ -0,0 +1,101 @@
+
+# Getting Started with LanceDB: A Minimal Vector Search Tutorial
+
+Let's set up a LanceDB database, insert vector data, and perform a simple vector search. We'll use simple character classes like "knight" and "rogue" to illustrate semantic relevance.
+
+## 1. Install Dependencies
+
+Before starting, make sure you have the necessary packages:
+
+```bash
+pip install lancedb pandas numpy
+```
+
+## 2. Import Required Libraries
+
+```python
+import lancedb
+import pandas as pd
+import numpy as np
+```
+
+## 3. Connect to LanceDB
+
+You can use a local directory to store your database:
+
+```python
+db = lancedb.connect("./lancedb")
+```
+
+## 4. Create Sample Data
+
+Add sample text data and corresponding 4D vectors:
+
+```python
+data = pd.DataFrame([
+    {"id": "1", "vector": [1.0, 0.0, 0.0, 0.0], "text": "knight"},
+    {"id": "2", "vector": [0.9, 0.1, 0.0, 0.0], "text": "warrior"},
+    {"id": "3", "vector": [0.0, 1.0, 0.0, 0.0], "text": "rogue"},
+    {"id": "4", "vector": [0.0, 0.9, 0.1, 0.0], "text": "thief"},
+    {"id": "5", "vector": [0.5, 0.5, 0.0, 0.0], "text": "ranger"},
+])
+```
+
+## 5. Create a Table in LanceDB
+
+```python
+table = db.create_table("rpg_classes", data=data, mode="overwrite")
+```
+
+Let's see how the table looks:
+```python
+print(data)
+```
+
+| id | vector | text |
+|----|--------|------|
+| 1 | [1.0, 0.0, 0.0, 0.0] | knight |
+| 2 | [0.9, 0.1, 0.0, 0.0] | warrior |
+| 3 | [0.0, 1.0, 0.0, 0.0] | rogue |
+| 4 | [0.0, 0.9, 0.1, 0.0] | thief |
+| 5 | [0.5, 0.5, 0.0, 0.0] | ranger |
+
+
+
+## 6. Perform a Vector Search
+
+Search for the most similar character classes to our query vector:
+
+```python
+# Query as if we are searching for "rogue"
+results = table.search([0.95, 0.05, 0.0, 0.0]).limit(3).to_df()
+print(results)
+```
+
+This will return the top 3 closest classes to the vector, effectively showing how LanceDB can be used for semantic search.
+
+|  id  |          vector         |   text   | _distance |
+|------|------------------------|----------|-----------|
+|  3   | [0.0, 1.0, 0.0, 0.0]  |  rogue   |   0.00   |
+|  4   | [0.0, 0.9, 0.1, 0.0]  | thief |   0.02   |
+|  5   | [0.5, 0.5, 0.0, 0.0]  |  ranger  |   0.50   |
+
+Let's try searching for "knight"
+
+```python
+query_vector = [1.0, 0.0, 0.0, 0.0]
+results = table.search(query_vector).limit(3).to_pandas()
+print(results)
+```
+
+|  id  |          vector         |   text   | _distance |
+|------|------------------------|----------|-----------|
+|  1   | [1.0, 0.0, 0.0, 0.0]  |  knight  |   0.00   |
+|  2   | [0.9, 0.1, 0.0, 0.0]  | warrior  |   0.02   |
+|  5   | [0.5, 0.5, 0.0, 0.0]  |  ranger  |   0.50   |
+
+## Next Steps
+
+That's it - you just conducted vector search!
+
+For more beginner tips, check out the [Basic Usage](basic.md) guide.

docs/src/troubleshooting.md

@@ -35,3 +35,9 @@ print the resolved query plan. You can use the `explain_plan` method to do this:
 * Python Sync: [LanceQueryBuilder.explain_plan][lancedb.query.LanceQueryBuilder.explain_plan]
 * Python Async: [AsyncQueryBase.explain_plan][lancedb.query.AsyncQueryBase.explain_plan]
 * Node @lancedb/lancedb: [LanceQueryBuilder.explainPlan](/lancedb/js/classes/QueryBase/#explainplan)
+
+To understand how a query was actually executed—including metrics like execution time, number of rows processed, I/O stats, and more—use the analyze_plan method. This executes the query and returns a physical execution plan annotated with runtime metrics, making it especially helpful for performance tuning and debugging.
+
+* Python Sync: [LanceQueryBuilder.analyze_plan][lancedb.query.LanceQueryBuilder.analyze_plan]
+* Python Async: [AsyncQueryBase.analyze_plan][lancedb.query.AsyncQueryBase.analyze_plan]
+* Node @lancedb/lancedb: [LanceQueryBuilder.analyzePlan](/lancedb/js/classes/QueryBase/#analyzePlan)

java/core/pom.xml

@@ -8,13 +8,16 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.18.2-beta.1</version>
+        <version>0.19.1-beta.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
     <artifactId>lancedb-core</artifactId>
     <name>LanceDB Core</name>
     <packaging>jar</packaging>
+    <properties>
+        <rust.release.build>false</rust.release.build>
+    </properties>
 
     <dependencies>
         <dependency>
@@ -68,7 +71,7 @@
                                 </goals>
                                 <configuration>
                                     <path>lancedb-jni</path>
-                                    <release>true</release>
+                                    <release>${rust.release.build}</release>
                                     <!-- Copy native libraries to target/classes for runtime access -->
                                     <copyTo>${project.build.directory}/classes/nativelib</copyTo>
                                     <copyWithPlatformDir>true</copyWithPlatformDir>

java/core/src/main/java/com/lancedb/lancedb/Connection.java

@@ -1,16 +1,25 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.lancedb.lancedb;
 
 import io.questdb.jar.jni.JarJniLoader;
+
 import java.io.Closeable;
 import java.util.List;
 import java.util.Optional;
 
-/**
- * Represents LanceDB database.
- */
+/** Represents LanceDB database. */
 public class Connection implements Closeable {
   static {
     JarJniLoader.loadLib(Connection.class, "/nativelib", "lancedb_jni");
@@ -18,14 +27,11 @@ public class Connection implements Closeable {
 
   private long nativeConnectionHandle;
 
-  /**
-   * Connect to a LanceDB instance.
-   */
+  /** Connect to a LanceDB instance. */
   public static native Connection connect(String uri);
 
   /**
-   * Get the names of all tables in the database. The names are sorted in
-   * ascending order.
+   * Get the names of all tables in the database. The names are sorted in ascending order.
    *
    * @return the table names
    */
@@ -34,8 +40,7 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
-   * ascending order.
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
    *
    * @param limit The number of results to return.
    * @return the table names
@@ -45,12 +50,11 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
-   * ascending order.
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
-   *                   by setting this to the last table name from the previous page.
+   *     value. This can be combined with limit to implement pagination by setting this to the last
+   *     table name from the previous page.
    * @return the table names
    */
   public List<String> tableNames(String startAfter) {
@@ -58,12 +62,11 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
-   * ascending order.
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
-   *                   by setting this to the last table name from the previous page.
+   *     value. This can be combined with limit to implement pagination by setting this to the last
+   *     table name from the previous page.
    * @param limit The number of results to return.
    * @return the table names
    */
@@ -72,22 +75,19 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Get the names of filtered tables in the database. The names are sorted in
-   * ascending order.
+   * Get the names of filtered tables in the database. The names are sorted in ascending order.
    *
    * @param startAfter If present, only return names that come lexicographically after the supplied
-   *                   value. This can be combined with limit to implement pagination
-   *                   by setting this to the last table name from the previous page.
+   *     value. This can be combined with limit to implement pagination by setting this to the last
+   *     table name from the previous page.
    * @param limit The number of results to return.
    * @return the table names
    */
-  public native List<String> tableNames(
-      Optional<String> startAfter, Optional<Integer> limit);
+  public native List<String> tableNames(Optional<String> startAfter, Optional<Integer> limit);
 
   /**
-   * Closes this connection and releases any system resources associated with it. If
-   * the connection is
-   * already closed, then invoking this method has no effect.
+   * Closes this connection and releases any system resources associated with it. If the connection
+   * is already closed, then invoking this method has no effect.
    */
   @Override
   public void close() {
@@ -98,8 +98,7 @@ public class Connection implements Closeable {
   }
 
   /**
-   * Native method to release the Lance connection resources associated with the
-   * given handle.
+   * Native method to release the Lance connection resources associated with the given handle.
    *
    * @param handle The native handle to the connection resource.
    */

java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java

@@ -1,27 +1,35 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.lancedb.lancedb;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.nio.file.Path;
-import java.util.List;
-import java.net.URL;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 public class ConnectionTest {
   private static final String[] TABLE_NAMES = {
-      "dataset_version",
-      "new_empty_dataset",
-      "test",
-      "write_stream"
+    "dataset_version", "new_empty_dataset", "test", "write_stream"
   };
 
-  @TempDir
-  static Path tempDir; // Temporary directory for the tests
+  @TempDir static Path tempDir; // Temporary directory for the tests
   private static URL lanceDbURL;
 
   @BeforeAll
@@ -53,18 +61,21 @@ public class ConnectionTest {
   @Test
   void tableNamesStartAfter() {
     try (Connection conn = Connection.connect(lanceDbURL.toString())) {
-      assertTableNamesStartAfter(conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(
+          conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[1], 2, TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[2], 1, TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, TABLE_NAMES[3], 0);
-      assertTableNamesStartAfter(conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(
+          conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "o_dataset", 2, TABLE_NAMES[2], TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "v_dataset", 1, TABLE_NAMES[3]);
       assertTableNamesStartAfter(conn, "z_dataset", 0);
     }
   }
 
-  private void assertTableNamesStartAfter(Connection conn, String startAfter, int expectedSize, String... expectedNames) {
+  private void assertTableNamesStartAfter(
+      Connection conn, String startAfter, int expectedSize, String... expectedNames) {
     List<String> tableNames = conn.tableNames(startAfter);
     assertEquals(expectedSize, tableNames.size());
     for (int i = 0; i < expectedNames.length; i++) {
@@ -74,7 +85,7 @@ public class ConnectionTest {
 
   @Test
   void tableNamesLimit() {
-      try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+    try (Connection conn = Connection.connect(lanceDbURL.toString())) {
       for (int i = 0; i <= TABLE_NAMES.length; i++) {
         List<String> tableNames = conn.tableNames(i);
         assertEquals(i, tableNames.size());

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.18.2-beta.1</version>
+    <version>0.19.1-beta.1</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>
@@ -29,6 +29,25 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
+        <spotless.skip>false</spotless.skip>
+        <spotless.version>2.30.0</spotless.version>
+        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+        <spotless.delimiter>package</spotless.delimiter>
+        <spotless.license.header>
+            /*
+            * Licensed under the Apache License, Version 2.0 (the "License");
+            * you may not use this file except in compliance with the License.
+            * You may obtain a copy of the License at
+            *
+            *     http://www.apache.org/licenses/LICENSE-2.0
+            *
+            * Unless required by applicable law or agreed to in writing, software
+            * distributed under the License is distributed on an "AS IS" BASIS,
+            * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+            * See the License for the specific language governing permissions and
+            * limitations under the License.
+            */
+        </spotless.license.header>
     </properties>
 
     <modules>
@@ -127,7 +146,8 @@
                 <configuration>
                     <configLocation>google_checks.xml</configLocation>
                     <consoleOutput>true</consoleOutput>
-                    <failsOnError>true</failsOnError>
+                    <failsOnError>false</failsOnError>
+                    <failOnViolation>false</failOnViolation>
                     <violationSeverity>warning</violationSeverity>
                     <linkXRef>false</linkXRef>
                 </configuration>
@@ -141,6 +161,10 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>com.diffplug.spotless</groupId>
+                <artifactId>spotless-maven-plugin</artifactId>
+            </plugin>
         </plugins>
         <pluginManagement>
             <plugins>
@@ -179,6 +203,54 @@
                     <artifactId>maven-install-plugin</artifactId>
                     <version>2.5.2</version>
                 </plugin>
+                <plugin>
+                    <groupId>com.diffplug.spotless</groupId>
+                    <artifactId>spotless-maven-plugin</artifactId>
+                    <version>${spotless.version}</version>
+                    <configuration>
+                        <skip>${spotless.skip}</skip>
+                        <upToDateChecking>
+                            <enabled>true</enabled>
+                        </upToDateChecking>
+                        <java>
+                            <includes>
+                                <include>src/main/java/**/*.java</include>
+                                <include>src/test/java/**/*.java</include>
+                            </includes>
+                            <googleJavaFormat>
+                                <version>${spotless.java.googlejavaformat.version}</version>
+                                <style>GOOGLE</style>
+                            </googleJavaFormat>
+
+                            <importOrder>
+                                <order>com.lancedb.lance,,javax,java,\#</order>
+                            </importOrder>
+
+                            <removeUnusedImports />
+                        </java>
+                        <scala>
+                            <includes>
+                                <include>src/main/scala/**/*.scala</include>
+                                <include>src/main/scala-*/**/*.scala</include>
+                                <include>src/test/scala/**/*.scala</include>
+                                <include>src/test/scala-*/**/*.scala</include>
+                            </includes>
+                        </scala>
+                        <licenseHeader>
+                            <content>${spotless.license.header}</content>
+                            <delimiter>${spotless.delimiter}</delimiter>
+                        </licenseHeader>
+                    </configuration>
+                    <executions>
+                        <execution>
+                            <id>spotless-check</id>
+                            <phase>validate</phase>
+                            <goals>
+                                <goal>apply</goal>
+                            </goals>
+                        </execution>
+                    </executions>
+                </plugin>
             </plugins>
         </pluginManagement>
     </build>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.18.2-beta.0",
+  "version": "0.19.1-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.18.2-beta.0",
+      "version": "0.19.1-beta.1",
       "cpu": [
         "x64",
         "arm64"
@@ -52,11 +52,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.18.2-beta.0",
-        "@lancedb/vectordb-darwin-x64": "0.18.2-beta.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.18.2-beta.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.18.2-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.18.2-beta.0"
+        "@lancedb/vectordb-darwin-arm64": "0.19.1-beta.1",
+        "@lancedb/vectordb-darwin-x64": "0.19.1-beta.1",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.19.1-beta.1",
+        "@lancedb/vectordb-linux-x64-gnu": "0.19.1-beta.1",
+        "@lancedb/vectordb-win32-x64-msvc": "0.19.1-beta.1"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -327,9 +327,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.18.2-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.18.2-beta.0.tgz",
-      "integrity": "sha512-FzIcElkS6R5I5kU1S5m7yLVTB1Duv1XcmZQtVmYl/JjNlfxS1WTtMzdzMqSBFohDcgU2Tkc5+1FpK1B94dUUbg==",
+      "version": "0.19.1-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.19.1-beta.1.tgz",
+      "integrity": "sha512-Epvel0pF5TM6MtIWQ2KhqezqSSHTL3Wr7a2rGAwz6X/XY23i6DbMPpPs0HyeIDzDrhxNfE3cz3S+SiCA6xpR0g==",
       "cpu": [
         "arm64"
       ],
@@ -340,9 +340,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.18.2-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.18.2-beta.0.tgz",
-      "integrity": "sha512-jv+XludfLNBDm1DjdqyghwDMtd4E+ygwycQpkpK72wyZSh6Qytrgq+4dNi/zCZ3UChFLbKbIxrVxv9yENQn2Pg==",
+      "version": "0.19.1-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.19.1-beta.1.tgz",
+      "integrity": "sha512-hOiUSlIoISbiXytp46hToi/r6sF5pImAsfbzCsIq8ExDV4TPa8fjbhcIT80vxxOwc2mpSSK4HsVJYod95RSbEQ==",
       "cpu": [
         "x64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.18.2-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.18.2-beta.0.tgz",
-      "integrity": "sha512-8/fBpbNYhhpetf/pZv0DyPnQkeAbsiICMyCoRiNu5auvQK4AsGF1XvLWrDi68u9F0GysBKvuatYuGqa/yh+Anw==",
+      "version": "0.19.1-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.19.1-beta.1.tgz",
+      "integrity": "sha512-/1JhGVDEngwrlM8o2TNW8G6nJ9U/VgHKAORmj/cTA7O30helJIoo9jfvUAUy+vZ4VoEwRXQbMI+gaYTg0l3MTg==",
       "cpu": [
         "arm64"
       ],
@@ -366,9 +366,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.18.2-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.18.2-beta.0.tgz",
-      "integrity": "sha512-7a1Kc/2V2ff4HlLzXyXVdK0Z0VIFUt50v2SBRdlcycJ0NLW9ZqV+9UjB/NAOwMXVgYd7d3rKjACGkQzkpvcyeg==",
+      "version": "0.19.1-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.19.1-beta.1.tgz",
+      "integrity": "sha512-zNRGSSUt8nTJMmll4NdxhQjwxR8Rezq3T4dsRoiDts5ienMam5HFjYiZ3FkDZQo16rgq2BcbFuH1G8u1chywlg==",
       "cpu": [
         "x64"
       ],
@@ -379,9 +379,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.18.2-beta.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.18.2-beta.0.tgz",
-      "integrity": "sha512-EeCiSf2RtJMESnkIca28GI6rAStYj2q9sVIyNCXpmIZSkJVpfQ3iswHGAbHrEfaPl0J1Re9cnRHLLuqkumwiIQ==",
+      "version": "0.19.1-beta.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.19.1-beta.1.tgz",
+      "integrity": "sha512-yV550AJGlsIFdm1KoHQPJ1TZx121ZXCIdebBtBZj3wOObIhyB/i0kZAtGvwjkmr7EYyfzt1EHZzbjSGVdehIAA==",
       "cpu": [
         "x64"
       ],
@@ -1184,9 +1184,10 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.7.7",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
-      "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
+      "version": "1.8.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz",
+      "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==",
+      "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.18.2-beta.1",
+  "version": "0.19.1-beta.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "private": false,
   "main": "dist/index.js",
@@ -89,10 +89,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.18.2-beta.1",
-    "@lancedb/vectordb-darwin-arm64": "0.18.2-beta.1",
-    "@lancedb/vectordb-linux-x64-gnu": "0.18.2-beta.1",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.18.2-beta.1",
-    "@lancedb/vectordb-win32-x64-msvc": "0.18.2-beta.1"
+    "@lancedb/vectordb-darwin-x64": "0.19.1-beta.1",
+    "@lancedb/vectordb-darwin-arm64": "0.19.1-beta.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.19.1-beta.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.19.1-beta.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.19.1-beta.1"
   }
 }

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.18.2-beta.1"
+version = "0.19.1-beta.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -28,6 +28,9 @@ napi-derive = "2.16.4"
 lzma-sys = { version = "*", features = ["static"] }
 log.workspace = true
 
+# Workaround for build failure until we can fix it.
+aws-lc-sys = "=0.28.0"
+
 [build-dependencies]
 napi-build = "2.1"
 

nodejs/__test__/arrow.test.ts

@@ -374,6 +374,71 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         expect(table2.numRows).toBe(4);
         expect(table2.schema).toEqual(schema);
       });
+
+      it("should correctly retain values in nested struct fields", async function () {
+        // Define test data with nested struct
+        const testData = [
+          {
+            id: "doc1",
+            vector: [1, 2, 3],
+            metadata: {
+              filePath: "/path/to/file1.ts",
+              startLine: 10,
+              endLine: 20,
+              text: "function test() { return true; }",
+            },
+          },
+          {
+            id: "doc2",
+            vector: [4, 5, 6],
+            metadata: {
+              filePath: "/path/to/file2.ts",
+              startLine: 30,
+              endLine: 40,
+              text: "function test2() { return false; }",
+            },
+          },
+        ];
+
+        // Create Arrow table from the data
+        const table = makeArrowTable(testData);
+
+        // Verify schema has the nested struct fields
+        const metadataField = table.schema.fields.find(
+          (f) => f.name === "metadata",
+        );
+        expect(metadataField).toBeDefined();
+        // biome-ignore lint/suspicious/noExplicitAny: accessing fields in different Arrow versions
+        const childNames = metadataField?.type.children.map((c: any) => c.name);
+        expect(childNames).toEqual([
+          "filePath",
+          "startLine",
+          "endLine",
+          "text",
+        ]);
+
+        // Convert to buffer and back (simulating storage and retrieval)
+        const buf = await fromTableToBuffer(table);
+        const retrievedTable = tableFromIPC(buf);
+
+        // Verify the retrieved table has the same structure
+        const rows = [];
+        for (let i = 0; i < retrievedTable.numRows; i++) {
+          rows.push(retrievedTable.get(i));
+        }
+
+        // Check values in the first row
+        const firstRow = rows[0];
+        expect(firstRow.id).toBe("doc1");
+        expect(firstRow.vector.toJSON()).toEqual([1, 2, 3]);
+
+        // Verify metadata values are preserved (this is where the bug is)
+        expect(firstRow.metadata).toBeDefined();
+        expect(firstRow.metadata.filePath).toBe("/path/to/file1.ts");
+        expect(firstRow.metadata.startLine).toBe(10);
+        expect(firstRow.metadata.endLine).toBe(20);
+        expect(firstRow.metadata.text).toBe("function test() { return true; }");
+      });
     });
 
     class DummyEmbedding extends EmbeddingFunction<string> {

nodejs/__test__/table.test.ts

@@ -10,7 +10,7 @@ import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
-import { Table, connect } from "../lancedb";
+import { MatchQuery, PhraseQuery, Table, connect } from "../lancedb";
 import {
   Table as ArrowTable,
   Field,
@@ -33,6 +33,7 @@ import {
   register,
 } from "../lancedb/embedding";
 import { Index } from "../lancedb/indices";
+import { instanceOfFullTextQuery } from "../lancedb/query";
 
 describe.each([arrow15, arrow16, arrow17, arrow18])(
   "Given a table",
@@ -70,6 +71,29 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await expect(table.countRows()).resolves.toBe(3);
     });
 
+    it("should show table stats", async () => {
+      await table.add([{ id: 1 }, { id: 2 }]);
+      await table.add([{ id: 1 }]);
+      await expect(table.stats()).resolves.toEqual({
+        fragmentStats: {
+          lengths: {
+            max: 2,
+            mean: 1,
+            min: 1,
+            p25: 1,
+            p50: 2,
+            p75: 2,
+            p99: 2,
+          },
+          numFragments: 2,
+          numSmallFragments: 2,
+        },
+        numIndices: 0,
+        numRows: 3,
+        totalBytes: 24,
+      });
+    });
+
     it("should overwrite data if asked", async () => {
       await table.add([{ id: 1 }, { id: 2 }]);
       await table.add([{ id: 1 }], { mode: "overwrite" });
@@ -314,11 +338,16 @@ describe("merge insert", () => {
       { a: 3, b: "y" },
       { a: 4, b: "z" },
     ];
-    await table
+    const stats = await table
       .mergeInsert("a")
       .whenMatchedUpdateAll()
       .whenNotMatchedInsertAll()
       .execute(newData);
+
+    expect(stats.numInsertedRows).toBe(1n);
+    expect(stats.numUpdatedRows).toBe(2n);
+    expect(stats.numDeletedRows).toBe(0n);
+
     const expected = [
       { a: 1, b: "a" },
       { a: 2, b: "x" },
@@ -506,6 +535,15 @@ describe("When creating an index", () => {
     expect(indices2.length).toBe(0);
   });
 
+  it("should wait for index readiness", async () => {
+    // Create an index and then wait for it to be ready
+    await tbl.createIndex("vec");
+    const indices = await tbl.listIndices();
+    expect(indices.length).toBeGreaterThan(0);
+    const idxName = indices[0].name;
+    await expect(tbl.waitForIndex([idxName], 5)).resolves.toBeUndefined();
+  });
+
   it("should search with distance range", async () => {
     await tbl.createIndex("vec");
 
@@ -633,6 +671,23 @@ describe("When creating an index", () => {
     expect(plan2).not.toMatch("LanceScan");
   });
 
+  it("should be able to run analyze plan", async () => {
+    await tbl.createIndex("vec");
+    await tbl.add([
+      {
+        id: 300,
+        vec: Array(32)
+          .fill(1)
+          .map(() => Math.random()),
+        tags: [],
+      },
+    ]);
+
+    const plan = await tbl.query().nearestTo(queryVec).analyzePlan();
+    expect(plan).toMatch("AnalyzeExec");
+    expect(plan).toMatch("metrics=");
+  });
+
   it("should be able to query with row id", async () => {
     const results = await tbl
       .query()
@@ -806,6 +861,7 @@ describe("When creating an index", () => {
     // Only build index over v1
     await tbl.createIndex("vec", {
       config: Index.ivfPq({ numPartitions: 2, numSubVectors: 2 }),
+      waitTimeoutSeconds: 30,
     });
 
     const rst = await tbl
@@ -850,6 +906,44 @@ describe("When creating an index", () => {
   });
 });
 
+describe("When querying a table", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should throw an error when timeout is reached", async () => {
+    const db = await connect(tmpDir.name);
+    const data = makeArrowTable([
+      { text: "a", vector: [0.1, 0.2] },
+      { text: "b", vector: [0.3, 0.4] },
+    ]);
+    const table = await db.createTable("test", data);
+    await table.createIndex("text", { config: Index.fts() });
+
+    await expect(
+      table.query().where("text != 'a'").toArray({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table.query().nearestTo([0.0, 0.0]).toArrow({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table.search("a", "fts").toArray({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+
+    await expect(
+      table
+        .query()
+        .nearestToText("a")
+        .nearestTo([0.0, 0.0])
+        .toArrow({ timeoutMs: 0 }),
+    ).rejects.toThrow("Query timeout");
+  });
+});
+
 describe("Read consistency interval", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -1112,6 +1206,73 @@ describe("when dealing with versioning", () => {
   });
 });
 
+describe("when dealing with tags", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("can manage tags", async () => {
+    const conn = await connect(tmpDir.name, {
+      readConsistencyInterval: 0,
+    });
+
+    const table = await conn.createTable("my_table", [
+      { id: 1n, vector: [0.1, 0.2] },
+    ]);
+    expect(await table.version()).toBe(1);
+
+    await table.add([{ id: 2n, vector: [0.3, 0.4] }]);
+    expect(await table.version()).toBe(2);
+
+    const tagsManager = await table.tags();
+
+    const initialTags = await tagsManager.list();
+    expect(Object.keys(initialTags).length).toBe(0);
+
+    const tag1 = "tag1";
+    await tagsManager.create(tag1, 1);
+    expect(await tagsManager.getVersion(tag1)).toBe(1);
+
+    const tagsAfterFirst = await tagsManager.list();
+    expect(Object.keys(tagsAfterFirst).length).toBe(1);
+    expect(tagsAfterFirst).toHaveProperty(tag1);
+    expect(tagsAfterFirst[tag1].version).toBe(1);
+
+    await tagsManager.create("tag2", 2);
+    expect(await tagsManager.getVersion("tag2")).toBe(2);
+
+    const tagsAfterSecond = await tagsManager.list();
+    expect(Object.keys(tagsAfterSecond).length).toBe(2);
+    expect(tagsAfterSecond).toHaveProperty(tag1);
+    expect(tagsAfterSecond[tag1].version).toBe(1);
+    expect(tagsAfterSecond).toHaveProperty("tag2");
+    expect(tagsAfterSecond["tag2"].version).toBe(2);
+
+    await table.add([{ id: 3n, vector: [0.5, 0.6] }]);
+    await tagsManager.update(tag1, 3);
+    expect(await tagsManager.getVersion(tag1)).toBe(3);
+
+    await tagsManager.delete("tag2");
+    const tagsAfterDelete = await tagsManager.list();
+    expect(Object.keys(tagsAfterDelete).length).toBe(1);
+    expect(tagsAfterDelete).toHaveProperty(tag1);
+    expect(tagsAfterDelete[tag1].version).toBe(3);
+
+    await table.add([{ id: 4n, vector: [0.7, 0.8] }]);
+    expect(await table.version()).toBe(4);
+
+    await table.checkout(tag1);
+    expect(await table.version()).toBe(3);
+
+    await table.checkoutLatest();
+    expect(await table.version()).toBe(4);
+  });
+});
+
 describe("when optimizing a dataset", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;
@@ -1247,6 +1408,56 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
 
       const results = await table.search("hello").toArray();
       expect(results[0].text).toBe(data[0].text);
+
+      const query = new MatchQuery("goodbye", "text");
+      expect(instanceOfFullTextQuery(query)).toBe(true);
+      const results2 = await table
+        .search(new MatchQuery("goodbye", "text"))
+        .toArray();
+      expect(results2[0].text).toBe(data[1].text);
+    });
+
+    test("prewarm full text search index", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: ["lance database", "the", "search"], vector: [0.1, 0.2, 0.3] },
+        { text: ["lance database"], vector: [0.4, 0.5, 0.6] },
+        { text: ["lance", "search"], vector: [0.7, 0.8, 0.9] },
+        { text: ["database", "search"], vector: [1.0, 1.1, 1.2] },
+        { text: ["unrelated", "doc"], vector: [1.3, 1.4, 1.5] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      // For the moment, we just confirm we can call prewarmIndex without error
+      // and still search it afterwards
+      await table.prewarmIndex("text_idx");
+
+      const results = await table.search("lance").toArray();
+      expect(results.length).toBe(3);
+    });
+
+    test("full text index on list", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: ["lance database", "the", "search"], vector: [0.1, 0.2, 0.3] },
+        { text: ["lance database"], vector: [0.4, 0.5, 0.6] },
+        { text: ["lance", "search"], vector: [0.7, 0.8, 0.9] },
+        { text: ["database", "search"], vector: [1.0, 1.1, 1.2] },
+        { text: ["unrelated", "doc"], vector: [1.3, 1.4, 1.5] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      const results = await table.search("lance").toArray();
+      expect(results.length).toBe(3);
+
+      const results2 = await table.search('"lance database"').toArray();
+      expect(results2.length).toBe(2);
     });
 
     test("full text search without positions", async () => {
@@ -1299,6 +1510,43 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results.length).toBe(2);
       const phraseResults = await table.search('"hello world"').toArray();
       expect(phraseResults.length).toBe(1);
+      const phraseResults2 = await table
+        .search(new PhraseQuery("hello world", "text"))
+        .toArray();
+      expect(phraseResults2.length).toBe(1);
+    });
+
+    test("full text search fuzzy query", async () => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "fa", vector: [0.1, 0.2, 0.3] },
+        { text: "fo", vector: [0.4, 0.5, 0.6] },
+        { text: "fob", vector: [0.4, 0.5, 0.6] },
+        { text: "focus", vector: [0.4, 0.5, 0.6] },
+        { text: "foo", vector: [0.4, 0.5, 0.6] },
+        { text: "food", vector: [0.4, 0.5, 0.6] },
+        { text: "foul", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+      await table.createIndex("text", {
+        config: Index.fts(),
+      });
+
+      const results = await table
+        .search(new MatchQuery("foo", "text"))
+        .toArray();
+      expect(results.length).toBe(1);
+      expect(results[0].text).toBe("foo");
+
+      const fuzzyResults = await table
+        .search(new MatchQuery("foo", "text", { fuzziness: 1 }))
+        .toArray();
+      expect(fuzzyResults.length).toBe(4);
+      const resultSet = new Set(fuzzyResults.map((r) => r.text));
+      expect(resultSet.has("foo")).toBe(true);
+      expect(resultSet.has("fob")).toBe(true);
+      expect(resultSet.has("fo")).toBe(true);
+      expect(resultSet.has("food")).toBe(true);
     });
 
     test.each([
@@ -1346,6 +1594,30 @@ describe("when calling explainPlan", () => {
   });
 });
 
+describe("when calling analyzePlan", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  let queryVec: number[];
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const con = await connect(tmpDir.name);
+    table = await con.createTable("vectors", [{ id: 1, vector: [1.1, 0.9] }]);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("retrieves runtime metrics", async () => {
+    queryVec = Array(2)
+      .fill(1)
+      .map(() => Math.random());
+    const plan = await table.query().nearestTo(queryVec).analyzePlan();
+    console.log("Query Plan:\n", plan); // <--- Print the plan
+    expect(plan).toMatch("AnalyzeExec");
+  });
+});
+
 describe("column name options", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;

nodejs/lancedb/arrow.ts

@@ -639,8 +639,9 @@ function transposeData(
 ): Vector {
   if (field.type instanceof Struct) {
     const childFields = field.type.children;
+    const fullPath = [...path, field.name];
     const childVectors = childFields.map((child) => {
-      return transposeData(data, child, [...path, child.name]);
+      return transposeData(data, child, fullPath);
     });
     const structData = makeData({
       type: field.type,
@@ -652,7 +653,14 @@ function transposeData(
     const values = data.map((datum) => {
       let current: unknown = datum;
       for (const key of valuesPath) {
-        if (isObject(current) && Object.hasOwn(current, key)) {
+        if (current == null) {
+          return null;
+        }
+
+        if (
+          isObject(current) &&
+          (Object.hasOwn(current, key) || key in current)
+        ) {
           current = current[key];
         } else {
           return null;

nodejs/lancedb/index.ts

@@ -23,6 +23,12 @@ export {
   OptimizeStats,
   CompactionStats,
   RemovalStats,
+  TableStatistics,
+  FragmentStatistics,
+  FragmentSummaryStats,
+  Tags,
+  TagContents,
+  MergeStats,
 } from "./native.js";
 
 export {
@@ -47,6 +53,12 @@ export {
   QueryExecutionOptions,
   FullTextSearchOptions,
   RecordBatchIterator,
+  FullTextQuery,
+  MatchQuery,
+  PhraseQuery,
+  BoostQuery,
+  MultiMatchQuery,
+  FullTextQueryType,
 } from "./query";
 
 export {

nodejs/lancedb/indices.ts

@@ -681,4 +681,6 @@ export interface IndexOptions {
    * The default is true
    */
   replace?: boolean;
+
+  waitTimeoutSeconds?: number;
 }

nodejs/lancedb/merge.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import { Data, Schema, fromDataToBuffer } from "./arrow";
-import { NativeMergeInsertBuilder } from "./native";
+import { MergeStats, NativeMergeInsertBuilder } from "./native";
 
 /** A builder used to create and run a merge insert operation */
 export class MergeInsertBuilder {
@@ -73,9 +73,9 @@ export class MergeInsertBuilder {
   /**
    * Executes the merge insert operation
    *
-   * Nothing is returned but the `Table` is updated
+   * @returns Statistics about the merge operation: counts of inserted, updated, and deleted rows
    */
-  async execute(data: Data): Promise<void> {
+  async execute(data: Data): Promise<MergeStats> {
     let schema: Schema;
     if (this.#schema instanceof Promise) {
       schema = await this.#schema;
@@ -84,6 +84,6 @@ export class MergeInsertBuilder {
       schema = this.#schema;
     }
     const buffer = await fromDataToBuffer(data, undefined, schema);
-    await this.#native.execute(buffer);
+    return await this.#native.execute(buffer);
   }
 }

nodejs/lancedb/query.ts

@@ -11,12 +11,14 @@ import {
 } from "./arrow";
 import { type IvfPqOptions } from "./indices";
 import {
+  JsFullTextQuery,
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
 import { Reranker } from "./rerankers";
+
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -62,7 +64,7 @@ class RecordBatchIterable<
   // biome-ignore lint/suspicious/noExplicitAny: skip
   [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
     return new RecordBatchIterator(
-      this.inner.execute(this.options?.maxBatchLength),
+      this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs),
     );
   }
 }
@@ -78,6 +80,11 @@ export interface QueryExecutionOptions {
    * in smaller chunks.
    */
   maxBatchLength?: number;
+
+  /**
+   * Timeout for query execution in milliseconds
+   */
+  timeoutMs?: number;
 }
 
 /**
@@ -152,7 +159,7 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
   }
 
   fullTextSearch(
-    query: string,
+    query: string | FullTextQuery,
     options?: Partial<FullTextSearchOptions>,
   ): this {
     let columns: string[] | null = null;
@@ -164,9 +171,16 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
       }
     }
 
-    this.doCall((inner: NativeQueryType) =>
-      inner.fullTextSearch(query, columns),
-    );
+    this.doCall((inner: NativeQueryType) => {
+      if (typeof query === "string") {
+        inner.fullTextSearch({
+          query: query,
+          columns: columns,
+        });
+      } else {
+        inner.fullTextSearch({ query: query.inner });
+      }
+    });
     return this;
   }
 
@@ -273,9 +287,11 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
     options?: Partial<QueryExecutionOptions>,
   ): Promise<NativeBatchIterator> {
     if (this.inner instanceof Promise) {
-      return this.inner.then((inner) => inner.execute(options?.maxBatchLength));
+      return this.inner.then((inner) =>
+        inner.execute(options?.maxBatchLength, options?.timeoutMs),
+      );
     } else {
-      return this.inner.execute(options?.maxBatchLength);
+      return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
     }
   }
 
@@ -348,6 +364,43 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
       return this.inner.explainPlan(verbose);
     }
   }
+
+  /**
+   * Executes the query and returns the physical query plan annotated with runtime metrics.
+   *
+   * This is useful for debugging and performance analysis, as it shows how the query was executed
+   * and includes metrics such as elapsed time, rows processed, and I/O statistics.
+   *
+   * @example
+   * import * as lancedb from "@lancedb/lancedb"
+   *
+   * const db = await lancedb.connect("./.lancedb");
+   * const table = await db.createTable("my_table", [
+   *   { vector: [1.1, 0.9], id: "1" },
+   * ]);
+   *
+   * const plan = await table.query().nearestTo([0.5, 0.2]).analyzePlan();
+   *
+   * Example output (with runtime metrics inlined):
+   * AnalyzeExec verbose=true, metrics=[]
+   *  ProjectionExec: expr=[id@3 as id, vector@0 as vector, _distance@2 as _distance], metrics=[output_rows=1, elapsed_compute=3.292µs]
+   *   Take: columns="vector, _rowid, _distance, (id)", metrics=[output_rows=1, elapsed_compute=66.001µs, batches_processed=1, bytes_read=8, iops=1, requests=1]
+   *    CoalesceBatchesExec: target_batch_size=1024, metrics=[output_rows=1, elapsed_compute=3.333µs]
+   *     GlobalLimitExec: skip=0, fetch=10, metrics=[output_rows=1, elapsed_compute=167ns]
+   *      FilterExec: _distance@2 IS NOT NULL, metrics=[output_rows=1, elapsed_compute=8.542µs]
+   *       SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=63.25µs, row_replacements=1]
+   *        KNNVectorDistance: metric=l2, metrics=[output_rows=1, elapsed_compute=114.333µs, output_batches=1]
+   *         LanceScan: uri=/path/to/data, projection=[vector], row_id=true, row_addr=false, ordered=false, metrics=[output_rows=1, elapsed_compute=103.626µs, bytes_read=549, iops=2, requests=2]
+   *
+   * @returns A query execution plan with runtime metrics for each step.
+   */
+  async analyzePlan(): Promise<string> {
+    if (this.inner instanceof Promise) {
+      return this.inner.then((inner) => inner.analyzePlan());
+    } else {
+      return this.inner.analyzePlan();
+    }
+  }
 }
 
 /**
@@ -681,8 +734,177 @@ export class Query extends QueryBase<NativeQuery> {
     }
   }
 
-  nearestToText(query: string, columns?: string[]): Query {
-    this.doCall((inner) => inner.fullTextSearch(query, columns));
+  nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
+    this.doCall((inner) => {
+      if (typeof query === "string") {
+        inner.fullTextSearch({
+          query: query,
+          columns: columns,
+        });
+      } else {
+        inner.fullTextSearch({ query: query.inner });
+      }
+    });
     return this;
   }
 }
+
+/**
+ * Enum representing the types of full-text queries supported.
+ *
+ * - `Match`: Performs a full-text search for terms in the query string.
+ * - `MatchPhrase`: Searches for an exact phrase match in the text.
+ * - `Boost`: Boosts the relevance score of specific terms in the query.
+ * - `MultiMatch`: Searches across multiple fields for the query terms.
+ */
+export enum FullTextQueryType {
+  Match = "match",
+  MatchPhrase = "match_phrase",
+  Boost = "boost",
+  MultiMatch = "multi_match",
+}
+
+/**
+ * Represents a full-text query interface.
+ * This interface defines the structure and behavior for full-text queries,
+ * including methods to retrieve the query type and convert the query to a dictionary format.
+ */
+export interface FullTextQuery {
+  /**
+   * Returns the inner query object.
+   * This is the underlying query object used by the database engine.
+   * @ignore
+   */
+  inner: JsFullTextQuery;
+
+  /**
+   * The type of the full-text query.
+   */
+  queryType(): FullTextQueryType;
+}
+
+// biome-ignore lint/suspicious/noExplicitAny: we want any here
+export function instanceOfFullTextQuery(obj: any): obj is FullTextQuery {
+  return obj != null && obj.inner instanceof JsFullTextQuery;
+}
+
+export class MatchQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of MatchQuery.
+   *
+   * @param query - The text query to search for.
+   * @param column - The name of the column to search within.
+   * @param options - Optional parameters for the match query.
+   *   - `boost`: The boost factor for the query (default is 1.0).
+   *   - `fuzziness`: The fuzziness level for the query (default is 0).
+   *   - `maxExpansions`: The maximum number of terms to consider for fuzzy matching (default is 50).
+   */
+  constructor(
+    query: string,
+    column: string,
+    options?: {
+      boost?: number;
+      fuzziness?: number;
+      maxExpansions?: number;
+    },
+  ) {
+    let fuzziness = options?.fuzziness;
+    if (fuzziness === undefined) {
+      fuzziness = 0;
+    }
+    this.inner = JsFullTextQuery.matchQuery(
+      query,
+      column,
+      options?.boost ?? 1.0,
+      fuzziness,
+      options?.maxExpansions ?? 50,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.Match;
+  }
+}
+
+export class PhraseQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of `PhraseQuery`.
+   *
+   * @param query - The phrase to search for in the specified column.
+   * @param column - The name of the column to search within.
+   */
+  constructor(query: string, column: string) {
+    this.inner = JsFullTextQuery.phraseQuery(query, column);
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.MatchPhrase;
+  }
+}
+
+export class BoostQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of BoostQuery.
+   * The boost returns documents that match the positive query,
+   * but penalizes those that match the negative query.
+   * the penalty is controlled by the `negativeBoost` parameter.
+   *
+   * @param positive - The positive query that boosts the relevance score.
+   * @param negative - The negative query that reduces the relevance score.
+   * @param options - Optional parameters for the boost query.
+   *  - `negativeBoost`: The boost factor for the negative query (default is 0.0).
+   */
+  constructor(
+    positive: FullTextQuery,
+    negative: FullTextQuery,
+    options?: {
+      negativeBoost?: number;
+    },
+  ) {
+    this.inner = JsFullTextQuery.boostQuery(
+      positive.inner,
+      negative.inner,
+      options?.negativeBoost,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.Boost;
+  }
+}
+
+export class MultiMatchQuery implements FullTextQuery {
+  /** @ignore */
+  public readonly inner: JsFullTextQuery;
+  /**
+   * Creates an instance of MultiMatchQuery.
+   *
+   * @param query - The text query to search for across multiple columns.
+   * @param columns - An array of column names to search within.
+   * @param options - Optional parameters for the multi-match query.
+   *  - `boosts`: An array of boost factors for each column (default is 1.0 for all).
+   */
+  constructor(
+    query: string,
+    columns: string[],
+    options?: {
+      boosts?: number[];
+    },
+  ) {
+    this.inner = JsFullTextQuery.multiMatchQuery(
+      query,
+      columns,
+      options?.boosts,
+    );
+  }
+
+  queryType(): FullTextQueryType {
+    return FullTextQueryType.MultiMatch;
+  }
+}

nodejs/lancedb/table.ts

@@ -20,9 +20,16 @@ import {
   IndexConfig,
   IndexStatistics,
   OptimizeStats,
+  TableStatistics,
+  Tags,
   Table as _NativeTable,
 } from "./native";
-import { Query, VectorQuery } from "./query";
+import {
+  FullTextQuery,
+  Query,
+  VectorQuery,
+  instanceOfFullTextQuery,
+} from "./query";
 import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
@@ -230,6 +237,30 @@ export abstract class Table {
    */
   abstract dropIndex(name: string): Promise<void>;
 
+  /**
+   * Prewarm an index in the table.
+   *
+   * @param name The name of the index.
+   *
+   * This will load the index into memory.  This may reduce the cold-start time for
+   * future queries.  If the index does not fit in the cache then this call may be
+   * wasteful.
+   */
+  abstract prewarmIndex(name: string): Promise<void>;
+
+  /**
+   * Waits for asynchronous indexing to complete on the table.
+   *
+   * @param indexNames The name of the indices to wait for
+   * @param timeoutSeconds The number of seconds to wait before timing out
+   *
+   * This will raise an error if the indices are not created and fully indexed within the timeout.
+   */
+  abstract waitForIndex(
+    indexNames: string[],
+    timeoutSeconds: number,
+  ): Promise<void>;
+
   /**
    * Create a {@link Query} Builder.
    *
@@ -294,7 +325,7 @@ export abstract class Table {
    * if the query is a string and no embedding function is defined, it will be treated as a full text search query
    */
   abstract search(
-    query: string | IntoVector,
+    query: string | IntoVector | FullTextQuery,
     queryType?: string,
     ftsColumns?: string | string[],
   ): VectorQuery | Query;
@@ -345,7 +376,7 @@ export abstract class Table {
    *
    * Calling this method will set the table into time-travel mode. If you
    * wish to return to standard mode, call `checkoutLatest`.
-   * @param {number} version The version to checkout
+   * @param {number | string} version The version to checkout, could be version number or tag
    * @example
    * ```typescript
    * import * as lancedb from "@lancedb/lancedb"
@@ -361,7 +392,8 @@ export abstract class Table {
    * console.log(await table.version()); // 2
    * ```
    */
-  abstract checkout(version: number): Promise<void>;
+  abstract checkout(version: number | string): Promise<void>;
+
   /**
    * Checkout the latest version of the table. _This is an in-place operation._
    *
@@ -375,6 +407,23 @@ export abstract class Table {
    */
   abstract listVersions(): Promise<Version[]>;
 
+  /**
+   * Get a tags manager for this table.
+   *
+   * Tags allow you to label specific versions of a table with a human-readable name.
+   * The returned tags manager can be used to list, create, update, or delete tags.
+   *
+   * @returns {Tags} A tags manager for this table
+   * @example
+   * ```typescript
+   * const tagsManager = await table.tags();
+   * await tagsManager.create("v1", 1);
+   * const tags = await tagsManager.list();
+   * console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
+   * ```
+   */
+  abstract tags(): Promise<Tags>;
+
   /**
    * Restore the table to the currently checked out version
    *
@@ -434,6 +483,13 @@ export abstract class Table {
    * Use {@link Table.listIndices} to find the names of the indices.
    */
   abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+
+  /** Returns table and fragment statistics
+   *
+   * @returns {TableStatistics} The table and fragment statistics
+   *
+   */
+  abstract stats(): Promise<TableStatistics>;
 }
 
 export class LocalTable extends Table {
@@ -553,23 +609,39 @@ export class LocalTable extends Table {
     // Bit of a hack to get around the fact that TS has no package-scope.
     // biome-ignore lint/suspicious/noExplicitAny: skip
     const nativeIndex = (options?.config as any)?.inner;
-    await this.inner.createIndex(nativeIndex, column, options?.replace);
+    await this.inner.createIndex(
+      nativeIndex,
+      column,
+      options?.replace,
+      options?.waitTimeoutSeconds,
+    );
   }
 
   async dropIndex(name: string): Promise<void> {
     await this.inner.dropIndex(name);
   }
 
+  async prewarmIndex(name: string): Promise<void> {
+    await this.inner.prewarmIndex(name);
+  }
+
+  async waitForIndex(
+    indexNames: string[],
+    timeoutSeconds: number,
+  ): Promise<void> {
+    await this.inner.waitForIndex(indexNames, timeoutSeconds);
+  }
+
   query(): Query {
     return new Query(this.inner);
   }
 
   search(
-    query: string | IntoVector,
+    query: string | IntoVector | FullTextQuery,
     queryType: string = "auto",
     ftsColumns?: string | string[],
   ): VectorQuery | Query {
-    if (typeof query !== "string") {
+    if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
       if (queryType === "fts") {
         throw new Error("Cannot perform full text search on a vector query");
       }
@@ -585,7 +657,10 @@ export class LocalTable extends Table {
 
     // The query type is auto or vector
     // fall back to full text search if no embedding functions are defined and the query is a string
-    if (queryType === "auto" && getRegistry().length() === 0) {
+    if (
+      queryType === "auto" &&
+      (getRegistry().length() === 0 || instanceOfFullTextQuery(query))
+    ) {
       return this.query().fullTextSearch(query, {
         columns: ftsColumns,
       });
@@ -651,8 +726,11 @@ export class LocalTable extends Table {
     return await this.inner.version();
   }
 
-  async checkout(version: number): Promise<void> {
-    await this.inner.checkout(version);
+  async checkout(version: number | string): Promise<void> {
+    if (typeof version === "string") {
+      return this.inner.checkoutTag(version);
+    }
+    return this.inner.checkout(version);
   }
 
   async checkoutLatest(): Promise<void> {
@@ -671,6 +749,10 @@ export class LocalTable extends Table {
     await this.inner.restore();
   }
 
+  async tags(): Promise<Tags> {
+    return await this.inner.tags();
+  }
+
   async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
     let cleanupOlderThanMs;
     if (
@@ -701,6 +783,11 @@ export class LocalTable extends Table {
     }
     return stats;
   }
+
+  async stats(): Promise<TableStatistics> {
+    return await this.inner.stats();
+  }
+
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
     return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.18.2-beta.1",
+  "version": "0.19.1-beta.1",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.18.2-beta.1",
+	"version": "0.19.1-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.18.2-beta.0",
+  "version": "0.19.1-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.18.2-beta.0",
+      "version": "0.19.1-beta.1",
       "cpu": [
         "x64",
         "arm64"
@@ -2304,89 +2304,20 @@
       }
     },
     "node_modules/@babel/code-frame": {
-      "version": "7.23.5",
-      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz",
-      "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==",
+      "version": "7.26.2",
+      "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz",
+      "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/highlight": "^7.23.4",
-        "chalk": "^2.4.2"
+        "@babel/helper-validator-identifier": "^7.25.9",
+        "js-tokens": "^4.0.0",
+        "picocolors": "^1.0.0"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/code-frame/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-      "dev": true,
-      "dependencies": {
-        "color-convert": "^1.9.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/code-frame/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-      "dev": true,
-      "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/code-frame/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-      "dev": true,
-      "dependencies": {
-        "color-name": "1.1.3"
-      }
-    },
-    "node_modules/@babel/code-frame/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-      "dev": true
-    },
-    "node_modules/@babel/code-frame/node_modules/escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.8.0"
-      }
-    },
-    "node_modules/@babel/code-frame/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/code-frame/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/@babel/compat-data": {
       "version": "7.23.5",
       "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.5.tgz",
@@ -2589,19 +2520,21 @@
       }
     },
     "node_modules/@babel/helper-string-parser": {
-      "version": "7.23.4",
-      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz",
-      "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz",
+      "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=6.9.0"
       }
     },
     "node_modules/@babel/helper-validator-identifier": {
-      "version": "7.22.20",
-      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
-      "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz",
+      "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=6.9.0"
       }
@@ -2616,109 +2549,28 @@
       }
     },
     "node_modules/@babel/helpers": {
-      "version": "7.23.8",
-      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.8.tgz",
-      "integrity": "sha512-KDqYz4PiOWvDFrdHLPhKtCThtIcKVy6avWD2oG4GEvyQ+XDZwHD4YQd+H2vNMnq2rkdxsDkU82T+Vk8U/WXHRQ==",
+      "version": "7.27.0",
+      "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.0.tgz",
+      "integrity": "sha512-U5eyP/CTFPuNE3qk+WZMxFkp/4zUzdceQlfzf7DdGdhp+Fezd7HD+i8Y24ZuTMKX3wQBld449jijbGq6OdGNQg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/template": "^7.22.15",
-        "@babel/traverse": "^7.23.7",
-        "@babel/types": "^7.23.6"
+        "@babel/template": "^7.27.0",
+        "@babel/types": "^7.27.0"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@babel/highlight": {
-      "version": "7.23.4",
-      "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz",
-      "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==",
-      "dev": true,
-      "dependencies": {
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "chalk": "^2.4.2",
-        "js-tokens": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/ansi-styles": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-      "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-      "dev": true,
-      "dependencies": {
-        "color-convert": "^1.9.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/chalk": {
-      "version": "2.4.2",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-      "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-      "dev": true,
-      "dependencies": {
-        "ansi-styles": "^3.2.1",
-        "escape-string-regexp": "^1.0.5",
-        "supports-color": "^5.3.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/color-convert": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-      "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-      "dev": true,
-      "dependencies": {
-        "color-name": "1.1.3"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-      "dev": true
-    },
-    "node_modules/@babel/highlight/node_modules/escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.8.0"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/@babel/highlight/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/@babel/parser": {
-      "version": "7.23.6",
-      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.6.tgz",
-      "integrity": "sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==",
+      "version": "7.27.0",
+      "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.0.tgz",
+      "integrity": "sha512-iaepho73/2Pz7w2eMS0Q5f83+0RKI7i4xmiYeBmDzfRVbQtTOG7Ts0S4HzJVsTMGI9keU8rNfuZr8DKfSt7Yyg==",
       "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@babel/types": "^7.27.0"
+      },
       "bin": {
         "parser": "bin/babel-parser.js"
       },
@@ -2904,14 +2756,15 @@
       }
     },
     "node_modules/@babel/template": {
-      "version": "7.22.15",
-      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
-      "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
+      "version": "7.27.0",
+      "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.0.tgz",
+      "integrity": "sha512-2ncevenBqXI6qRMukPlXwHKHchC7RyMuu4xv5JBXRfOGVcTy1mXCD12qrp7Jsoxll1EV3+9sE4GugBVRjT2jFA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/code-frame": "^7.22.13",
-        "@babel/parser": "^7.22.15",
-        "@babel/types": "^7.22.15"
+        "@babel/code-frame": "^7.26.2",
+        "@babel/parser": "^7.27.0",
+        "@babel/types": "^7.27.0"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -2948,14 +2801,14 @@
       }
     },
     "node_modules/@babel/types": {
-      "version": "7.23.6",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.6.tgz",
-      "integrity": "sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==",
+      "version": "7.27.0",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.0.tgz",
+      "integrity": "sha512-H45s8fVLYjbhFH62dIJ3WtmJ6RSPt/3DRO0ZcT2SUiYiQyz3BLVb9ADEnLl91m74aQPS3AzzeajZHYOalWe3bg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/helper-string-parser": "^7.23.4",
-        "@babel/helper-validator-identifier": "^7.22.20",
-        "to-fast-properties": "^2.0.0"
+        "@babel/helper-string-parser": "^7.25.9",
+        "@babel/helper-validator-identifier": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -5550,10 +5403,11 @@
       "devOptional": true
     },
     "node_modules/axios": {
-      "version": "1.7.7",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
-      "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
+      "version": "1.8.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.4.tgz",
+      "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
@@ -7869,7 +7723,8 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
       "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/js-yaml": {
       "version": "3.14.1",
@@ -9360,15 +9215,6 @@
       "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==",
       "dev": true
     },
-    "node_modules/to-fast-properties": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
-      "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==",
-      "dev": true,
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.18.2-beta.1",
+  "version": "0.19.1-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -29,6 +29,7 @@
         "aarch64-apple-darwin",
         "x86_64-unknown-linux-gnu",
         "aarch64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl",
         "aarch64-unknown-linux-musl",
         "x86_64-pc-windows-msvc",
         "aarch64-pc-windows-msvc"

nodejs/src/merge.rs

@@ -37,7 +37,7 @@ impl NativeMergeInsertBuilder {
     }
 
     #[napi(catch_unwind)]
-    pub async fn execute(&self, buf: Buffer) -> napi::Result<()> {
+    pub async fn execute(&self, buf: Buffer) -> napi::Result<MergeStats> {
         let data = ipc_file_to_batches(buf.to_vec())
             .and_then(IntoArrow::into_arrow)
             .map_err(|e| {
@@ -46,12 +46,14 @@ impl NativeMergeInsertBuilder {
 
         let this = self.clone();
 
-        this.inner.execute(data).await.map_err(|e| {
+        let stats = this.inner.execute(data).await.map_err(|e| {
             napi::Error::from_reason(format!(
                 "Failed to execute merge insert: {}",
                 convert_error(&e)
             ))
-        })
+        })?;
+
+        Ok(stats.into())
     }
 }
 
@@ -60,3 +62,20 @@ impl From<MergeInsertBuilder> for NativeMergeInsertBuilder {
         Self { inner }
     }
 }
+
+#[napi(object)]
+pub struct MergeStats {
+    pub num_inserted_rows: BigInt,
+    pub num_updated_rows: BigInt,
+    pub num_deleted_rows: BigInt,
+}
+
+impl From<lancedb::table::MergeStats> for MergeStats {
+    fn from(stats: lancedb::table::MergeStats) -> Self {
+        Self {
+            num_inserted_rows: stats.num_inserted_rows.into(),
+            num_updated_rows: stats.num_updated_rows.into(),
+            num_deleted_rows: stats.num_deleted_rows.into(),
+        }
+    }
+}

nodejs/src/query.rs

@@ -3,7 +3,9 @@
 
 use std::sync::Arc;
 
-use lancedb::index::scalar::FullTextSearchQuery;
+use lancedb::index::scalar::{
+    BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, PhraseQuery,
+};
 use lancedb::query::ExecutableQuery;
 use lancedb::query::Query as LanceDbQuery;
 use lancedb::query::QueryBase;
@@ -38,9 +40,10 @@ impl Query {
     }
 
     #[napi]
-    pub fn full_text_search(&mut self, query: String, columns: Option<Vec<String>>) {
-        let query = FullTextSearchQuery::new(query).columns(columns);
+    pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
+        let query = parse_fts_query(query)?;
         self.inner = self.inner.clone().full_text_search(query);
+        Ok(())
     }
 
     #[napi]
@@ -87,11 +90,15 @@ impl Query {
     pub async fn execute(
         &self,
         max_batch_length: Option<u32>,
+        timeout_ms: Option<u32>,
     ) -> napi::Result<RecordBatchIterator> {
         let mut execution_opts = QueryExecutionOptions::default();
         if let Some(max_batch_length) = max_batch_length {
             execution_opts.max_batch_length = max_batch_length;
         }
+        if let Some(timeout_ms) = timeout_ms {
+            execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
+        }
         let inner_stream = self
             .inner
             .execute_with_options(execution_opts)
@@ -114,6 +121,16 @@ impl Query {
             ))
         })
     }
+
+    #[napi(catch_unwind)]
+    pub async fn analyze_plan(&self) -> napi::Result<String> {
+        self.inner.analyze_plan().await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to execute analyze plan: {}",
+                convert_error(&e)
+            ))
+        })
+    }
 }
 
 #[napi]
@@ -185,9 +202,10 @@ impl VectorQuery {
     }
 
     #[napi]
-    pub fn full_text_search(&mut self, query: String, columns: Option<Vec<String>>) {
-        let query = FullTextSearchQuery::new(query).columns(columns);
+    pub fn full_text_search(&mut self, query: napi::JsObject) -> napi::Result<()> {
+        let query = parse_fts_query(query)?;
         self.inner = self.inner.clone().full_text_search(query);
+        Ok(())
     }
 
     #[napi]
@@ -232,11 +250,15 @@ impl VectorQuery {
     pub async fn execute(
         &self,
         max_batch_length: Option<u32>,
+        timeout_ms: Option<u32>,
     ) -> napi::Result<RecordBatchIterator> {
         let mut execution_opts = QueryExecutionOptions::default();
         if let Some(max_batch_length) = max_batch_length {
             execution_opts.max_batch_length = max_batch_length;
         }
+        if let Some(timeout_ms) = timeout_ms {
+            execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
+        }
         let inner_stream = self
             .inner
             .execute_with_options(execution_opts)
@@ -259,4 +281,127 @@ impl VectorQuery {
             ))
         })
     }
+
+    #[napi(catch_unwind)]
+    pub async fn analyze_plan(&self) -> napi::Result<String> {
+        self.inner.analyze_plan().await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to execute analyze plan: {}",
+                convert_error(&e)
+            ))
+        })
+    }
+}
+
+#[napi]
+#[derive(Debug, Clone)]
+pub struct JsFullTextQuery {
+    pub(crate) inner: FtsQuery,
+}
+
+#[napi]
+impl JsFullTextQuery {
+    #[napi(factory)]
+    pub fn match_query(
+        query: String,
+        column: String,
+        boost: f64,
+        fuzziness: Option<u32>,
+        max_expansions: u32,
+    ) -> napi::Result<Self> {
+        Ok(Self {
+            inner: MatchQuery::new(query)
+                .with_column(Some(column))
+                .with_boost(boost as f32)
+                .with_fuzziness(fuzziness)
+                .with_max_expansions(max_expansions as usize)
+                .into(),
+        })
+    }
+
+    #[napi(factory)]
+    pub fn phrase_query(query: String, column: String) -> napi::Result<Self> {
+        Ok(Self {
+            inner: PhraseQuery::new(query).with_column(Some(column)).into(),
+        })
+    }
+
+    #[napi(factory)]
+    #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
+    pub fn boost_query(
+        positive: &JsFullTextQuery,
+        negative: &JsFullTextQuery,
+        negative_boost: Option<f64>,
+    ) -> napi::Result<Self> {
+        Ok(Self {
+            inner: BoostQuery::new(
+                positive.inner.clone(),
+                negative.inner.clone(),
+                negative_boost.map(|v| v as f32),
+            )
+            .into(),
+        })
+    }
+
+    #[napi(factory)]
+    pub fn multi_match_query(
+        query: String,
+        columns: Vec<String>,
+        boosts: Option<Vec<f64>>,
+    ) -> napi::Result<Self> {
+        let q = match boosts {
+            Some(boosts) => MultiMatchQuery::try_new(query, columns)
+                .and_then(|q| q.try_with_boosts(boosts.into_iter().map(|v| v as f32).collect())),
+            None => MultiMatchQuery::try_new(query, columns),
+        }
+        .map_err(|e| {
+            napi::Error::from_reason(format!("Failed to create multi match query: {}", e))
+        })?;
+
+        Ok(Self { inner: q.into() })
+    }
+}
+
+fn parse_fts_query(query: napi::JsObject) -> napi::Result<FullTextSearchQuery> {
+    if let Ok(Some(query)) = query.get::<_, &JsFullTextQuery>("query") {
+        Ok(FullTextSearchQuery::new_query(query.inner.clone()))
+    } else if let Ok(Some(query_text)) = query.get::<_, String>("query") {
+        let mut query_text = query_text;
+        let columns = query.get::<_, Option<Vec<String>>>("columns")?.flatten();
+
+        let is_phrase =
+            query_text.len() >= 2 && query_text.starts_with('"') && query_text.ends_with('"');
+        let is_multi_match = columns.as_ref().map(|cols| cols.len() > 1).unwrap_or(false);
+
+        if is_phrase {
+            // Remove the surrounding quotes for phrase queries
+            query_text = query_text[1..query_text.len() - 1].to_string();
+        }
+
+        let query: FtsQuery = match (is_phrase, is_multi_match) {
+            (false, _) => MatchQuery::new(query_text).into(),
+            (true, false) => PhraseQuery::new(query_text).into(),
+            (true, true) => {
+                return Err(napi::Error::from_reason(
+                    "Phrase queries cannot be used with multiple columns.",
+                ));
+            }
+        };
+        let mut query = FullTextSearchQuery::new_query(query);
+        if let Some(cols) = columns {
+            if !cols.is_empty() {
+                query = query.with_columns(&cols).map_err(|e| {
+                    napi::Error::from_reason(format!(
+                        "Failed to set full text search columns: {}",
+                        e
+                    ))
+                })?;
+            }
+        }
+        Ok(query)
+    } else {
+        Err(napi::Error::from_reason(
+            "Invalid full text search query object".to_string(),
+        ))
+    }
 }

nodejs/src/table.rs

@@ -111,6 +111,7 @@ impl Table {
         index: Option<&Index>,
         column: String,
         replace: Option<bool>,
+        wait_timeout_s: Option<i64>,
     ) -> napi::Result<()> {
         let lancedb_index = if let Some(index) = index {
             index.consume()?
@@ -121,6 +122,10 @@ impl Table {
         if let Some(replace) = replace {
             builder = builder.replace(replace);
         }
+        if let Some(timeout) = wait_timeout_s {
+            builder =
+                builder.wait_timeout(std::time::Duration::from_secs(timeout.try_into().unwrap()));
+        }
         builder.execute().await.default_error()
     }
 
@@ -132,6 +137,32 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn prewarm_index(&self, index_name: String) -> napi::Result<()> {
+        self.inner_ref()?
+            .prewarm_index(&index_name)
+            .await
+            .default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn wait_for_index(&self, index_names: Vec<String>, timeout_s: i64) -> Result<()> {
+        let timeout = std::time::Duration::from_secs(timeout_s.try_into().unwrap());
+        let index_names: Vec<&str> = index_names.iter().map(|s| s.as_str()).collect();
+        let slice: &[&str] = &index_names;
+
+        self.inner_ref()?
+            .wait_for_index(slice, timeout)
+            .await
+            .default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn stats(&self) -> Result<TableStatistics> {
+        let stats = self.inner_ref()?.stats().await.default_error()?;
+        Ok(stats.into())
+    }
+
     #[napi(catch_unwind)]
     pub async fn update(
         &self,
@@ -224,6 +255,14 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn checkout_tag(&self, tag: String) -> napi::Result<()> {
+        self.inner_ref()?
+            .checkout_tag(tag.as_str())
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn checkout_latest(&self) -> napi::Result<()> {
         self.inner_ref()?.checkout_latest().await.default_error()
@@ -256,6 +295,13 @@ impl Table {
         self.inner_ref()?.restore().await.default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn tags(&self) -> napi::Result<Tags> {
+        Ok(Tags {
+            inner: self.inner_ref()?.clone(),
+        })
+    }
+
     #[napi(catch_unwind)]
     pub async fn optimize(
         &self,
@@ -515,9 +561,158 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
     }
 }
 
+#[napi(object)]
+pub struct TableStatistics {
+    /// The total number of bytes in the table
+    pub total_bytes: i64,
+
+    /// The number of rows in the table
+    pub num_rows: i64,
+
+    /// The number of indices in the table
+    pub num_indices: i64,
+
+    /// Statistics on table fragments
+    pub fragment_stats: FragmentStatistics,
+}
+
+#[napi(object)]
+pub struct FragmentStatistics {
+    /// The number of fragments in the table
+    pub num_fragments: i64,
+
+    /// The number of uncompacted fragments in the table
+    pub num_small_fragments: i64,
+
+    /// Statistics on the number of rows in the table fragments
+    pub lengths: FragmentSummaryStats,
+}
+
+#[napi(object)]
+pub struct FragmentSummaryStats {
+    /// The number of rows in the fragment with the fewest rows
+    pub min: i64,
+
+    /// The number of rows in the fragment with the most rows
+    pub max: i64,
+
+    /// The mean number of rows in the fragments
+    pub mean: i64,
+
+    /// The 25th percentile of number of rows in the fragments
+    pub p25: i64,
+
+    /// The 50th percentile of number of rows in the fragments
+    pub p50: i64,
+
+    /// The 75th percentile of number of rows in the fragments
+    pub p75: i64,
+
+    /// The 99th percentile of number of rows in the fragments
+    pub p99: i64,
+}
+
+impl From<lancedb::table::TableStatistics> for TableStatistics {
+    fn from(v: lancedb::table::TableStatistics) -> Self {
+        Self {
+            total_bytes: v.total_bytes as i64,
+            num_rows: v.num_rows as i64,
+            num_indices: v.num_indices as i64,
+            fragment_stats: FragmentStatistics {
+                num_fragments: v.fragment_stats.num_fragments as i64,
+                num_small_fragments: v.fragment_stats.num_small_fragments as i64,
+                lengths: FragmentSummaryStats {
+                    min: v.fragment_stats.lengths.min as i64,
+                    max: v.fragment_stats.lengths.max as i64,
+                    mean: v.fragment_stats.lengths.mean as i64,
+                    p25: v.fragment_stats.lengths.p25 as i64,
+                    p50: v.fragment_stats.lengths.p50 as i64,
+                    p75: v.fragment_stats.lengths.p75 as i64,
+                    p99: v.fragment_stats.lengths.p99 as i64,
+                },
+            },
+        }
+    }
+}
+
 #[napi(object)]
 pub struct Version {
     pub version: i64,
     pub timestamp: i64,
     pub metadata: HashMap<String, String>,
 }
+
+#[napi]
+pub struct TagContents {
+    pub version: i64,
+    pub manifest_size: i64,
+}
+
+#[napi]
+pub struct Tags {
+    inner: LanceDbTable,
+}
+
+#[napi]
+impl Tags {
+    #[napi]
+    pub async fn list(&self) -> napi::Result<HashMap<String, TagContents>> {
+        let rust_tags = self.inner.tags().await.default_error()?;
+        let tag_list = rust_tags.as_ref().list().await.default_error()?;
+        let tag_contents = tag_list
+            .into_iter()
+            .map(|(k, v)| {
+                (
+                    k,
+                    TagContents {
+                        version: v.version as i64,
+                        manifest_size: v.manifest_size as i64,
+                    },
+                )
+            })
+            .collect();
+
+        Ok(tag_contents)
+    }
+
+    #[napi]
+    pub async fn get_version(&self, tag: String) -> napi::Result<i64> {
+        let rust_tags = self.inner.tags().await.default_error()?;
+        rust_tags
+            .as_ref()
+            .get_version(tag.as_str())
+            .await
+            .map(|v| v as i64)
+            .default_error()
+    }
+
+    #[napi]
+    pub async unsafe fn create(&mut self, tag: String, version: i64) -> napi::Result<()> {
+        let mut rust_tags = self.inner.tags().await.default_error()?;
+        rust_tags
+            .as_mut()
+            .create(tag.as_str(), version as u64)
+            .await
+            .default_error()
+    }
+
+    #[napi]
+    pub async unsafe fn delete(&mut self, tag: String) -> napi::Result<()> {
+        let mut rust_tags = self.inner.tags().await.default_error()?;
+        rust_tags
+            .as_mut()
+            .delete(tag.as_str())
+            .await
+            .default_error()
+    }
+
+    #[napi]
+    pub async unsafe fn update(&mut self, tag: String, version: i64) -> napi::Result<()> {
+        let mut rust_tags = self.inner.tags().await.default_error()?;
+        rust_tags
+            .as_mut()
+            .update(tag.as_str(), version as u64)
+            .await
+            .default_error()
+    }
+}

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.21.2"
+current_version = "0.22.1-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.21.2"
+version = "0.22.1-beta.1"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/pyproject.toml

@@ -4,11 +4,12 @@ name = "lancedb"
 dynamic = ["version"]
 dependencies = [
     "deprecation",
-    "tqdm>=4.27.0",
-    "pyarrow>=14",
-    "pydantic>=1.10",
-    "packaging",
+    "numpy",
     "overrides>=0.7",
+    "packaging",
+    "pyarrow>=16",
+    "pydantic>=1.10",
+    "tqdm>=4.27.0",
 ]
 description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -42,6 +43,9 @@ classifiers = [
 repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
+pylance = [
+    "pylance>=0.25",
+]
 tests = [
     "aiohttp",
     "boto3",
@@ -54,7 +58,8 @@ tests = [
     "polars>=0.19, <=1.3.0",
     "tantivy",
     "pyarrow-stubs",
-    "pylance>=0.23.2",
+    "pylance>=0.25",
+    "requests",
 ]
 dev = [
     "ruff",
@@ -72,6 +77,7 @@ embeddings = [
     "pillow",
     "open-clip-torch",
     "cohere",
+    "colpali-engine>=0.3.10",
     "huggingface_hub",
     "InstructorEmbedding",
     "google.generativeai",

python/python/lancedb/_lancedb.pyi

@@ -1,4 +1,5 @@
-from typing import Dict, List, Optional, Tuple, Any, Union, Literal
+from datetime import timedelta
+from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
 
 import pyarrow as pa
 
@@ -46,12 +47,13 @@ class Table:
     ): ...
     async def list_versions(self) -> List[Dict[str, Any]]: ...
     async def version(self) -> int: ...
-    async def checkout(self, version: int): ...
+    async def checkout(self, version: Union[int, str]): ...
     async def checkout_latest(self): ...
-    async def restore(self): ...
+    async def restore(self, version: Optional[int] = None): ...
     async def list_indices(self) -> list[IndexConfig]: ...
     async def delete(self, filter: str): ...
     async def add_columns(self, columns: list[tuple[str, str]]) -> None: ...
+    async def add_columns_with_schema(self, schema: pa.Schema) -> None: ...
     async def alter_columns(self, columns: list[dict[str, Any]]) -> None: ...
     async def optimize(
         self,
@@ -59,9 +61,18 @@ class Table:
         cleanup_since_ms: Optional[int] = None,
         delete_unverified: Optional[bool] = None,
     ) -> OptimizeStats: ...
+    @property
+    def tags(self) -> Tags: ...
     def query(self) -> Query: ...
     def vector_search(self) -> VectorQuery: ...
 
+class Tags:
+    async def list(self) -> Dict[str, Tag]: ...
+    async def get_version(self, tag: str) -> int: ...
+    async def create(self, tag: str, version: int): ...
+    async def delete(self, tag: str): ...
+    async def update(self, tag: str, version: int): ...
+
 class IndexConfig:
     index_type: str
     columns: List[str]
@@ -93,7 +104,11 @@ class Query:
     def postfilter(self): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
     def nearest_to_text(self, query: dict) -> FTSQuery: ...
-    async def execute(self, max_batch_length: Optional[int]) -> RecordBatchStream: ...
+    async def execute(
+        self, max_batch_length: Optional[int], timeout: Optional[timedelta]
+    ) -> RecordBatchStream: ...
+    async def explain_plan(self, verbose: Optional[bool]) -> str: ...
+    async def analyze_plan(self) -> str: ...
     def to_query_request(self) -> PyQueryRequest: ...
 
 class FTSQuery:
@@ -107,8 +122,9 @@ class FTSQuery:
     def get_query(self) -> str: ...
     def add_query_vector(self, query_vec: pa.Array) -> None: ...
     def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
-    async def execute(self, max_batch_length: Optional[int]) -> RecordBatchStream: ...
-    async def explain_plan(self) -> str: ...
+    async def execute(
+        self, max_batch_length: Optional[int], timeout: Optional[timedelta]
+    ) -> RecordBatchStream: ...
     def to_query_request(self) -> PyQueryRequest: ...
 
 class VectorQuery:
@@ -188,3 +204,7 @@ class RemovalStats:
 class OptimizeStats:
     compaction: CompactionStats
     prune: RemovalStats
+
+class Tag(TypedDict):
+    version: int
+    manifest_size: int

python/python/lancedb/common.py

@@ -9,7 +9,7 @@ import numpy as np
 import pyarrow as pa
 import pyarrow.dataset
 
-from .dependencies import pandas as pd
+from .dependencies import _check_for_pandas, pandas as pd
 
 DATA = Union[List[dict], "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
 VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
@@ -63,7 +63,7 @@ def data_to_reader(
     data: DATA, schema: Optional[pa.Schema] = None
 ) -> pa.RecordBatchReader:
     """Convert various types of input into a RecordBatchReader"""
-    if pd is not None and isinstance(data, pd.DataFrame):
+    if _check_for_pandas(data) and isinstance(data, pd.DataFrame):
         return pa.Table.from_pandas(data, schema=schema).to_reader()
     elif isinstance(data, pa.Table):
         return data.to_reader()

python/python/lancedb/embeddings/__init__.py

@@ -19,3 +19,4 @@ from .imagebind import ImageBindEmbeddings
 from .jinaai import JinaEmbeddings
 from .watsonx import WatsonxEmbeddings
 from .voyageai import VoyageAIEmbeddingFunction
+from .colpali import ColPaliEmbeddings

python/python/lancedb/embeddings/colpali.py

@@ -0,0 +1,255 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+
+from functools import lru_cache
+from typing import List, Union, Optional, Any
+import numpy as np
+import io
+
+from ..util import attempt_import_or_raise
+from .base import EmbeddingFunction
+from .registry import register
+from .utils import TEXT, IMAGES, is_flash_attn_2_available
+
+
+@register("colpali")
+class ColPaliEmbeddings(EmbeddingFunction):
+    """
+    An embedding function that uses the ColPali engine for
+    multimodal multi-vector embeddings.
+
+    This embedding function supports ColQwen2.5 models, producing multivector outputs
+    for both text and image inputs. The output embeddings are lists of vectors, each
+    vector being 128-dimensional by default, represented as List[List[float]].
+
+    Parameters
+    ----------
+    model_name : str
+        The name of the model to use (e.g., "Metric-AI/ColQwen2.5-3b-multilingual-v1.0")
+    device : str
+        The device for inference (default "cuda:0").
+    dtype : str
+        Data type for model weights (default "bfloat16").
+    use_token_pooling : bool
+        Whether to use token pooling to reduce embedding size (default True).
+    pool_factor : int
+        Factor to reduce sequence length if token pooling is enabled (default 2).
+    quantization_config : Optional[BitsAndBytesConfig]
+        Quantization configuration for the model. (default None, bitsandbytes needed)
+    batch_size : int
+        Batch size for processing inputs (default 2).
+    """
+
+    model_name: str = "Metric-AI/ColQwen2.5-3b-multilingual-v1.0"
+    device: str = "auto"
+    dtype: str = "bfloat16"
+    use_token_pooling: bool = True
+    pool_factor: int = 2
+    quantization_config: Optional[Any] = None
+    batch_size: int = 2
+
+    _model = None
+    _processor = None
+    _token_pooler = None
+    _vector_dim = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        (
+            self._model,
+            self._processor,
+            self._token_pooler,
+        ) = self._load_model(
+            self.model_name,
+            self.dtype,
+            self.device,
+            self.use_token_pooling,
+            self.quantization_config,
+        )
+
+    @staticmethod
+    @lru_cache(maxsize=1)
+    def _load_model(
+        model_name: str,
+        dtype: str,
+        device: str,
+        use_token_pooling: bool,
+        quantization_config: Optional[Any],
+    ):
+        """
+        Initialize and cache the ColPali model, processor, and token pooler.
+        """
+        torch = attempt_import_or_raise("torch", "torch")
+        transformers = attempt_import_or_raise("transformers", "transformers")
+        colpali_engine = attempt_import_or_raise("colpali_engine", "colpali_engine")
+        from colpali_engine.compression.token_pooling import HierarchicalTokenPooler
+
+        if quantization_config is not None:
+            if not isinstance(quantization_config, transformers.BitsAndBytesConfig):
+                raise ValueError("quantization_config must be a BitsAndBytesConfig")
+
+        if dtype == "bfloat16":
+            torch_dtype = torch.bfloat16
+        elif dtype == "float16":
+            torch_dtype = torch.float16
+        elif dtype == "float64":
+            torch_dtype = torch.float64
+        else:
+            torch_dtype = torch.float32
+
+        model = colpali_engine.models.ColQwen2_5.from_pretrained(
+            model_name,
+            torch_dtype=torch_dtype,
+            device_map=device,
+            quantization_config=quantization_config
+            if quantization_config is not None
+            else None,
+            attn_implementation="flash_attention_2"
+            if is_flash_attn_2_available()
+            else None,
+        ).eval()
+        processor = colpali_engine.models.ColQwen2_5_Processor.from_pretrained(
+            model_name
+        )
+        token_pooler = HierarchicalTokenPooler() if use_token_pooling else None
+        return model, processor, token_pooler
+
+    def ndims(self):
+        """
+        Return the dimension of a vector in the multivector output (e.g., 128).
+        """
+        torch = attempt_import_or_raise("torch", "torch")
+        if self._vector_dim is None:
+            dummy_query = "test"
+            batch_queries = self._processor.process_queries([dummy_query]).to(
+                self._model.device
+            )
+            with torch.no_grad():
+                query_embeddings = self._model(**batch_queries)
+
+            if self.use_token_pooling and self._token_pooler is not None:
+                query_embeddings = self._token_pooler.pool_embeddings(
+                    query_embeddings,
+                    pool_factor=self.pool_factor,
+                    padding=True,
+                    padding_side=self._processor.tokenizer.padding_side,
+                )
+
+            self._vector_dim = query_embeddings[0].shape[-1]
+        return self._vector_dim
+
+    def _process_embeddings(self, embeddings):
+        """
+        Format model embeddings into List[List[float]].
+        Use token pooling if enabled.
+        """
+        torch = attempt_import_or_raise("torch", "torch")
+        if self.use_token_pooling and self._token_pooler is not None:
+            embeddings = self._token_pooler.pool_embeddings(
+                embeddings,
+                pool_factor=self.pool_factor,
+                padding=True,
+                padding_side=self._processor.tokenizer.padding_side,
+            )
+
+        if isinstance(embeddings, torch.Tensor):
+            tensors = embeddings.detach().cpu()
+            if tensors.dtype == torch.bfloat16:
+                tensors = tensors.to(torch.float32)
+            return (
+                tensors.numpy()
+                .astype(np.float64 if self.dtype == "float64" else np.float32)
+                .tolist()
+            )
+        return []
+
+    def generate_text_embeddings(self, text: TEXT) -> List[List[List[float]]]:
+        """
+        Generate embeddings for text input.
+        """
+        torch = attempt_import_or_raise("torch", "torch")
+        text = self.sanitize_input(text)
+        all_embeddings = []
+
+        for i in range(0, len(text), self.batch_size):
+            batch_text = text[i : i + self.batch_size]
+            batch_queries = self._processor.process_queries(batch_text).to(
+                self._model.device
+            )
+            with torch.no_grad():
+                query_embeddings = self._model(**batch_queries)
+            all_embeddings.extend(self._process_embeddings(query_embeddings))
+        return all_embeddings
+
+    def _prepare_images(self, images: IMAGES) -> List:
+        """
+        Convert image inputs to PIL Images.
+        """
+        PIL = attempt_import_or_raise("PIL", "pillow")
+        requests = attempt_import_or_raise("requests", "requests")
+        images = self.sanitize_input(images)
+        pil_images = []
+        try:
+            for image in images:
+                if isinstance(image, str):
+                    if image.startswith(("http://", "https://")):
+                        response = requests.get(image, timeout=10)
+                        response.raise_for_status()
+                        pil_images.append(PIL.Image.open(io.BytesIO(response.content)))
+                    else:
+                        with PIL.Image.open(image) as im:
+                            pil_images.append(im.copy())
+                elif isinstance(image, bytes):
+                    pil_images.append(PIL.Image.open(io.BytesIO(image)))
+                else:
+                    # Assume it's a PIL Image; will raise if invalid
+                    pil_images.append(image)
+        except Exception as e:
+            raise ValueError(f"Failed to process image: {e}")
+
+        return pil_images
+
+    def generate_image_embeddings(self, images: IMAGES) -> List[List[List[float]]]:
+        """
+        Generate embeddings for a batch of images.
+        """
+        torch = attempt_import_or_raise("torch", "torch")
+        pil_images = self._prepare_images(images)
+        all_embeddings = []
+
+        for i in range(0, len(pil_images), self.batch_size):
+            batch_images = pil_images[i : i + self.batch_size]
+            batch_images = self._processor.process_images(batch_images).to(
+                self._model.device
+            )
+            with torch.no_grad():
+                image_embeddings = self._model(**batch_images)
+            all_embeddings.extend(self._process_embeddings(image_embeddings))
+        return all_embeddings
+
+    def compute_query_embeddings(
+        self, query: Union[str, IMAGES], *args, **kwargs
+    ) -> List[List[List[float]]]:
+        """
+        Compute embeddings for a single user query (text only).
+        """
+        if not isinstance(query, str):
+            raise ValueError(
+                "Query must be a string, image to image search is not supported"
+            )
+        return self.generate_text_embeddings([query])
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[List[List[float]]]:
+        """
+        Compute embeddings for a batch of source images.
+
+        Parameters
+        ----------
+        images : Union[str, bytes, List, pa.Array, pa.ChunkedArray, np.ndarray]
+            Batch of images (paths, URLs, bytes, or PIL Images).
+        """
+        images = self.sanitize_input(images)
+        return self.generate_image_embeddings(images)

python/python/lancedb/embeddings/utils.py

@@ -18,6 +18,7 @@ import numpy as np
 import pyarrow as pa
 
 from ..dependencies import pandas as pd
+from ..util import attempt_import_or_raise
 
 
 # ruff: noqa: PERF203
@@ -275,3 +276,12 @@ def url_retrieve(url: str):
 def api_key_not_found_help(provider):
     logging.error("Could not find API key for %s", provider)
     raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")
+
+
+def is_flash_attn_2_available():
+    try:
+        attempt_import_or_raise("flash_attn", "flash_attn")
+
+        return True
+    except ImportError:
+        return False

python/python/lancedb/embeddings/voyageai.py

@@ -1,9 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-
+import base64
 import os
-from typing import ClassVar, TYPE_CHECKING, List, Union
+from typing import ClassVar, TYPE_CHECKING, List, Union, Any
+
+from pathlib import Path
+from urllib.parse import urlparse
+from io import BytesIO
 
 import numpy as np
 import pyarrow as pa
@@ -11,12 +14,100 @@ import pyarrow as pa
 from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
 from .registry import register
-from .utils import api_key_not_found_help, IMAGES
+from .utils import api_key_not_found_help, IMAGES, TEXT
 
 if TYPE_CHECKING:
     import PIL
 
 
+def is_valid_url(text):
+    try:
+        parsed = urlparse(text)
+        return bool(parsed.scheme) and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+def transform_input(input_data: Union[str, bytes, Path]):
+    PIL = attempt_import_or_raise("PIL", "pillow")
+    if isinstance(input_data, str):
+        if is_valid_url(input_data):
+            content = {"type": "image_url", "image_url": input_data}
+        else:
+            content = {"type": "text", "text": input_data}
+    elif isinstance(input_data, PIL.Image.Image):
+        buffered = BytesIO()
+        input_data.save(buffered, format="JPEG")
+        img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        content = {
+            "type": "image_base64",
+            "image_base64": "data:image/jpeg;base64," + img_str,
+        }
+    elif isinstance(input_data, bytes):
+        img = PIL.Image.open(BytesIO(input_data))
+        buffered = BytesIO()
+        img.save(buffered, format="JPEG")
+        img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        content = {
+            "type": "image_base64",
+            "image_base64": "data:image/jpeg;base64," + img_str,
+        }
+    elif isinstance(input_data, Path):
+        img = PIL.Image.open(input_data)
+        buffered = BytesIO()
+        img.save(buffered, format="JPEG")
+        img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        content = {
+            "type": "image_base64",
+            "image_base64": "data:image/jpeg;base64," + img_str,
+        }
+    else:
+        raise ValueError("Each input should be either str, bytes, Path or Image.")
+
+    return {"content": [content]}
+
+
+def sanitize_multimodal_input(inputs: Union[TEXT, IMAGES]) -> List[Any]:
+    """
+    Sanitize the input to the embedding function.
+    """
+    PIL = attempt_import_or_raise("PIL", "pillow")
+    if isinstance(inputs, (str, bytes, Path, PIL.Image.Image)):
+        inputs = [inputs]
+    elif isinstance(inputs, pa.Array):
+        inputs = inputs.to_pylist()
+    elif isinstance(inputs, pa.ChunkedArray):
+        inputs = inputs.combine_chunks().to_pylist()
+    else:
+        raise ValueError(
+            f"Input type {type(inputs)} not allowed with multimodal model."
+        )
+
+    if not all(isinstance(x, (str, bytes, Path, PIL.Image.Image)) for x in inputs):
+        raise ValueError("Each input should be either str, bytes, Path or Image.")
+
+    return [transform_input(i) for i in inputs]
+
+
+def sanitize_text_input(inputs: TEXT) -> List[str]:
+    """
+    Sanitize the input to the embedding function.
+    """
+    if isinstance(inputs, str):
+        inputs = [inputs]
+    elif isinstance(inputs, pa.Array):
+        inputs = inputs.to_pylist()
+    elif isinstance(inputs, pa.ChunkedArray):
+        inputs = inputs.combine_chunks().to_pylist()
+    else:
+        raise ValueError(f"Input type {type(inputs)} not allowed with text model.")
+
+    if not all(isinstance(x, str) for x in inputs):
+        raise ValueError("Each input should be str.")
+
+    return inputs
+
+
 @register("voyageai")
 class VoyageAIEmbeddingFunction(EmbeddingFunction):
     """
@@ -74,6 +165,11 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
     ]
     multimodal_embedding_models: list = ["voyage-multimodal-3"]
 
+    def _is_multimodal_model(self, model_name: str):
+        return (
+            model_name in self.multimodal_embedding_models or "multimodal" in model_name
+        )
+
     def ndims(self):
         if self.name == "voyage-3-lite":
             return 512
@@ -85,55 +181,12 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
             "voyage-finance-2",
             "voyage-multilingual-2",
             "voyage-law-2",
+            "voyage-multimodal-3",
         ]:
             return 1024
         else:
             raise ValueError(f"Model {self.name} not supported")
 
-    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(images, (str, bytes)):
-            images = [images]
-        elif isinstance(images, pa.Array):
-            images = images.to_pylist()
-        elif isinstance(images, pa.ChunkedArray):
-            images = images.combine_chunks().to_pylist()
-        return images
-
-    def generate_text_embeddings(self, text: str, **kwargs) -> np.ndarray:
-        """
-        Get the embeddings for the given texts
-
-        Parameters
-        ----------
-        texts: list[str] or np.ndarray (of str)
-            The texts to embed
-        input_type: Optional[str]
-
-        truncation: Optional[bool]
-        """
-        client = VoyageAIEmbeddingFunction._get_client()
-        if self.name in self.text_embedding_models:
-            rs = client.embed(texts=[text], model=self.name, **kwargs)
-        elif self.name in self.multimodal_embedding_models:
-            rs = client.multimodal_embed(inputs=[[text]], model=self.name, **kwargs)
-        else:
-            raise ValueError(
-                f"Model {self.name} not supported to generate text embeddings"
-            )
-
-        return rs.embeddings[0]
-
-    def generate_image_embedding(
-        self, image: "PIL.Image.Image", **kwargs
-    ) -> np.ndarray:
-        rs = VoyageAIEmbeddingFunction._get_client().multimodal_embed(
-            inputs=[[image]], model=self.name, **kwargs
-        )
-        return rs.embeddings[0]
-
     def compute_query_embeddings(
         self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
     ) -> List[np.ndarray]:
@@ -144,23 +197,52 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
         ----------
         query : Union[str, PIL.Image.Image]
             The query to embed. A query can be either text or an image.
+
+        Returns
+        -------
+            List[np.array]: the list of embeddings
         """
-        if isinstance(query, str):
-            return [self.generate_text_embeddings(query, input_type="query")]
+        client = VoyageAIEmbeddingFunction._get_client()
+        if self._is_multimodal_model(self.name):
+            result = client.multimodal_embed(
+                inputs=[[query]], model=self.name, input_type="query", **kwargs
+            )
         else:
-            PIL = attempt_import_or_raise("PIL", "pillow")
-            if isinstance(query, PIL.Image.Image):
-                return [self.generate_image_embedding(query, input_type="query")]
-            else:
-                raise TypeError("Only text PIL images supported as query")
+            result = client.embed(
+                texts=[query], model=self.name, input_type="query", **kwargs
+            )
+
+        return [result.embeddings[0]]
 
     def compute_source_embeddings(
-        self, images: IMAGES, *args, **kwargs
+        self, inputs: Union[TEXT, IMAGES], *args, **kwargs
     ) -> List[np.array]:
-        images = self.sanitize_input(images)
-        return [
-            self.generate_image_embedding(img, input_type="document") for img in images
-        ]
+        """
+        Compute the embeddings for the inputs
+
+        Parameters
+        ----------
+        inputs : Union[TEXT, IMAGES]
+            The inputs to embed. The input can be either str, bytes, Path (to an image),
+            PIL.Image or list of these.
+
+        Returns
+        -------
+            List[np.array]: the list of embeddings
+        """
+        client = VoyageAIEmbeddingFunction._get_client()
+        if self._is_multimodal_model(self.name):
+            inputs = sanitize_multimodal_input(inputs)
+            result = client.multimodal_embed(
+                inputs=inputs, model=self.name, input_type="document", **kwargs
+            )
+        else:
+            inputs = sanitize_text_input(inputs)
+            result = client.embed(
+                texts=inputs, model=self.name, input_type="document", **kwargs
+            )
+
+        return result.embeddings
 
     @staticmethod
     def _get_client():

python/python/lancedb/pydantic.py

@@ -152,6 +152,104 @@ def Vector(
     return FixedSizeList
 
 
+def MultiVector(
+    dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
+) -> Type:
+    """Pydantic MultiVector Type for multi-vector embeddings.
+
+    This type represents a list of vectors, each with the same dimension.
+    Useful for models that produce multiple embeddings per input, like ColPali.
+
+    Parameters
+    ----------
+    dim : int
+        The dimension of each vector in the multi-vector.
+    value_type : pyarrow.DataType, optional
+        The value type of the vectors, by default pa.float32()
+    nullable : bool, optional
+        Whether the multi-vector is nullable, by default it is True.
+
+    Examples
+    --------
+
+    >>> import pydantic
+    >>> from lancedb.pydantic import MultiVector
+    ...
+    >>> class MyModel(pydantic.BaseModel):
+    ...     id: int
+    ...     text: str
+    ...     embeddings: MultiVector(128)  # List of 128-dimensional vectors
+    >>> schema = pydantic_to_schema(MyModel)
+    >>> assert schema == pa.schema([
+    ...     pa.field("id", pa.int64(), False),
+    ...     pa.field("text", pa.utf8(), False),
+    ...     pa.field("embeddings", pa.list_(pa.list_(pa.float32(), 128)))
+    ... ])
+    """
+
+    class MultiVectorList(list, FixedSizeListMixin):
+        def __repr__(self):
+            return f"MultiVector(dim={dim})"
+
+        @staticmethod
+        def nullable() -> bool:
+            return nullable
+
+        @staticmethod
+        def dim() -> int:
+            return dim
+
+        @staticmethod
+        def value_arrow_type() -> pa.DataType:
+            return value_type
+
+        @staticmethod
+        def is_multi_vector() -> bool:
+            return True
+
+        @classmethod
+        def __get_pydantic_core_schema__(
+            cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler
+        ) -> CoreSchema:
+            return core_schema.no_info_after_validator_function(
+                cls,
+                core_schema.list_schema(
+                    items_schema=core_schema.list_schema(
+                        min_length=dim,
+                        max_length=dim,
+                        items_schema=core_schema.float_schema(),
+                    ),
+                ),
+            )
+
+        @classmethod
+        def __get_validators__(cls) -> Generator[Callable, None, None]:
+            yield cls.validate
+
+        # For pydantic v1
+        @classmethod
+        def validate(cls, v):
+            if not isinstance(v, (list, range)):
+                raise TypeError("A list of vectors is needed")
+            for vec in v:
+                if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim:
+                    raise TypeError(f"Each vector must be a list of {dim} numbers")
+            return cls(v)
+
+        if PYDANTIC_VERSION.major < 2:
+
+            @classmethod
+            def __modify_schema__(cls, field_schema: Dict[str, Any]):
+                field_schema["items"] = {
+                    "type": "array",
+                    "items": {"type": "number"},
+                    "minItems": dim,
+                    "maxItems": dim,
+                }
+
+    return MultiVectorList
+
+
 def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
     """Convert a field with native Python type to Arrow data type.
 
@@ -206,6 +304,9 @@ def _pydantic_type_to_arrow_type(tp: Any, field: FieldInfo) -> pa.DataType:
             fields = _pydantic_model_to_fields(tp)
             return pa.struct(fields)
         if issubclass(tp, FixedSizeListMixin):
+            if getattr(tp, "is_multi_vector", lambda: False)():
+                return pa.list_(pa.list_(tp.value_arrow_type(), tp.dim()))
+            # For regular Vector
             return pa.list_(tp.value_arrow_type(), tp.dim())
     return _py_type_to_arrow_type(tp, field)
 

python/python/lancedb/remote/table.py

@@ -18,7 +18,7 @@ from lancedb.merge import LanceMergeInsertBuilder
 from lancedb.embeddings import EmbeddingFunctionRegistry
 
 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder
-from ..table import AsyncTable, IndexStatistics, Query, Table
+from ..table import AsyncTable, IndexStatistics, Query, Table, Tags
 
 
 class RemoteTable(Table):
@@ -54,6 +54,10 @@ class RemoteTable(Table):
         """Get the current version of the table"""
         return LOOP.run(self._table.version())
 
+    @property
+    def tags(self) -> Tags:
+        return Tags(self._table)
+
     @cached_property
     def embedding_functions(self) -> Dict[str, EmbeddingFunctionConfig]:
         """
@@ -81,12 +85,15 @@ class RemoteTable(Table):
         """to_pandas() is not yet supported on LanceDB cloud."""
         return NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
 
-    def checkout(self, version: int):
+    def checkout(self, version: Union[int, str]):
         return LOOP.run(self._table.checkout(version))
 
     def checkout_latest(self):
         return LOOP.run(self._table.checkout_latest())
 
+    def restore(self, version: Optional[int] = None):
+        return LOOP.run(self._table.restore(version))
+
     def list_indices(self) -> Iterable[IndexConfig]:
         """List all the indices on the table"""
         return LOOP.run(self._table.list_indices())
@@ -101,6 +108,7 @@ class RemoteTable(Table):
         index_type: Literal["BTREE", "BITMAP", "LABEL_LIST", "scalar"] = "scalar",
         *,
         replace: bool = False,
+        wait_timeout: timedelta = None,
     ):
         """Creates a scalar index
         Parameters
@@ -123,13 +131,18 @@ class RemoteTable(Table):
         else:
             raise ValueError(f"Unknown index type: {index_type}")
 
-        LOOP.run(self._table.create_index(column, config=config, replace=replace))
+        LOOP.run(
+            self._table.create_index(
+                column, config=config, replace=replace, wait_timeout=wait_timeout
+            )
+        )
 
     def create_fts_index(
         self,
         column: str,
         *,
         replace: bool = False,
+        wait_timeout: timedelta = None,
         with_position: bool = True,
         # tokenizer configs:
         base_tokenizer: str = "simple",
@@ -150,7 +163,11 @@ class RemoteTable(Table):
             remove_stop_words=remove_stop_words,
             ascii_folding=ascii_folding,
         )
-        LOOP.run(self._table.create_index(column, config=config, replace=replace))
+        LOOP.run(
+            self._table.create_index(
+                column, config=config, replace=replace, wait_timeout=wait_timeout
+            )
+        )
 
     def create_index(
         self,
@@ -162,6 +179,7 @@ class RemoteTable(Table):
         replace: Optional[bool] = None,
         accelerator: Optional[str] = None,
         index_type="vector",
+        wait_timeout: Optional[timedelta] = None,
     ):
         """Create an index on the table.
         Currently, the only parameters that matter are
@@ -233,7 +251,11 @@ class RemoteTable(Table):
                 " 'IVF_FLAT', 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
             )
 
-        LOOP.run(self._table.create_index(vector_column_name, config=config))
+        LOOP.run(
+            self._table.create_index(
+                vector_column_name, config=config, wait_timeout=wait_timeout
+            )
+        )
 
     def add(
         self,
@@ -352,9 +374,15 @@ class RemoteTable(Table):
         )
 
     def _execute_query(
-        self, query: Query, batch_size: Optional[int] = None
+        self,
+        query: Query,
+        *,
+        batch_size: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
     ) -> pa.RecordBatchReader:
-        async_iter = LOOP.run(self._table._execute_query(query, batch_size=batch_size))
+        async_iter = LOOP.run(
+            self._table._execute_query(query, batch_size=batch_size, timeout=timeout)
+        )
 
         def iter_sync():
             try:
@@ -365,6 +393,12 @@ class RemoteTable(Table):
 
         return pa.RecordBatchReader.from_batches(async_iter.schema, iter_sync())
 
+    def _explain_plan(self, query: Query, verbose: Optional[bool] = False) -> str:
+        return LOOP.run(self._table._explain_plan(query, verbose))
+
+    def _analyze_plan(self, query: Query) -> str:
+        return LOOP.run(self._table._analyze_plan(query))
+
     def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
         """Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder]
         that can be used to create a "merge insert" operation.
@@ -539,6 +573,14 @@ class RemoteTable(Table):
     def drop_index(self, index_name: str):
         return LOOP.run(self._table.drop_index(index_name))
 
+    def wait_for_index(
+        self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)
+    ):
+        return LOOP.run(self._table.wait_for_index(index_names, timeout))
+
+    def stats(self):
+        return LOOP.run(self._table.stats())
+
     def uses_v2_manifest_paths(self) -> bool:
         raise NotImplementedError(
             "uses_v2_manifest_paths() is not supported on the LanceDB Cloud"

python/python/lancedb/rerankers/answerdotai.py

@@ -47,6 +47,9 @@ class AnswerdotaiRerankers(Reranker):
         )
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         docs = result_set[self.column].to_pylist()
         doc_ids = list(range(len(docs)))
         result = self.reranker.rank(query, docs, doc_ids=doc_ids)
@@ -83,7 +86,6 @@ class AnswerdotaiRerankers(Reranker):
         vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
             vector_results = vector_results.drop_columns(["_distance"])
-
         vector_results = vector_results.sort_by([("_relevance_score", "descending")])
         return vector_results
 
@@ -91,7 +93,5 @@ class AnswerdotaiRerankers(Reranker):
         fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
             fts_results = fts_results.drop_columns(["_score"])
-
         fts_results = fts_results.sort_by([("_relevance_score", "descending")])
-
         return fts_results

python/python/lancedb/rerankers/base.py

@@ -65,6 +65,16 @@ class Reranker(ABC):
             f"{self.__class__.__name__} does not implement rerank_vector"
         )
 
+    def _handle_empty_results(self, results: pa.Table):
+        """
+        Helper method to handle empty FTS results consistently
+        """
+        if len(results) > 0:
+            return results
+        return results.append_column(
+            "_relevance_score", pa.array([], type=pa.float32())
+        )
+
     def rerank_fts(
         self,
         query: str,

python/python/lancedb/rerankers/cohere.py

@@ -62,6 +62,9 @@ class CohereReranker(Reranker):
         return cohere.Client(os.environ.get("COHERE_API_KEY") or self.api_key)
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         docs = result_set[self.column].to_pylist()
         response = self._client.rerank(
             query=query,
@@ -99,24 +102,14 @@ class CohereReranker(Reranker):
             )
         return combined_results
 
-    def rerank_vector(
-        self,
-        query: str,
-        vector_results: pa.Table,
-    ):
-        result_set = self._rerank(vector_results, query)
+    def rerank_vector(self, query: str, vector_results: pa.Table):
+        vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_distance"])
+            vector_results = vector_results.drop_columns(["_distance"])
+        return vector_results
 
-        return result_set
-
-    def rerank_fts(
-        self,
-        query: str,
-        fts_results: pa.Table,
-    ):
-        result_set = self._rerank(fts_results, query)
+    def rerank_fts(self, query: str, fts_results: pa.Table):
+        fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_score"])
-
-        return result_set
+            fts_results = fts_results.drop_columns(["_score"])
+        return fts_results

python/python/lancedb/rerankers/cross_encoder.py

@@ -63,6 +63,9 @@ class CrossEncoderReranker(Reranker):
         return cross_encoder
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         passages = result_set[self.column].to_pylist()
         cross_inp = [[query, passage] for passage in passages]
         cross_scores = self.model.predict(cross_inp)
@@ -93,11 +96,7 @@ class CrossEncoderReranker(Reranker):
 
         return combined_results
 
-    def rerank_vector(
-        self,
-        query: str,
-        vector_results: pa.Table,
-    ):
+    def rerank_vector(self, query: str, vector_results: pa.Table):
         vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
             vector_results = vector_results.drop_columns(["_distance"])
@@ -105,11 +104,7 @@ class CrossEncoderReranker(Reranker):
         vector_results = vector_results.sort_by([("_relevance_score", "descending")])
         return vector_results
 
-    def rerank_fts(
-        self,
-        query: str,
-        fts_results: pa.Table,
-    ):
+    def rerank_fts(self, query: str, fts_results: pa.Table):
         fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
             fts_results = fts_results.drop_columns(["_score"])

python/python/lancedb/rerankers/jinaai.py

@@ -62,6 +62,9 @@ class JinaReranker(Reranker):
         return self._session
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         docs = result_set[self.column].to_pylist()
         response = self._client.post(  # type: ignore
             API_URL,
@@ -104,24 +107,14 @@ class JinaReranker(Reranker):
             )
         return combined_results
 
-    def rerank_vector(
-        self,
-        query: str,
-        vector_results: pa.Table,
-    ):
-        result_set = self._rerank(vector_results, query)
+    def rerank_vector(self, query: str, vector_results: pa.Table):
+        vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_distance"])
+            vector_results = vector_results.drop_columns(["_distance"])
+        return vector_results
 
-        return result_set
-
-    def rerank_fts(
-        self,
-        query: str,
-        fts_results: pa.Table,
-    ):
-        result_set = self._rerank(fts_results, query)
+    def rerank_fts(self, query: str, fts_results: pa.Table):
+        fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_score"])
-
-        return result_set
+            fts_results = fts_results.drop_columns(["_score"])
+        return fts_results

python/python/lancedb/rerankers/openai.py

@@ -44,6 +44,9 @@ class OpenaiReranker(Reranker):
         self.api_key = api_key
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         docs = result_set[self.column].to_pylist()
         response = self._client.chat.completions.create(
             model=self.model_name,
@@ -104,18 +107,14 @@ class OpenaiReranker(Reranker):
         vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
             vector_results = vector_results.drop_columns(["_distance"])
-
         vector_results = vector_results.sort_by([("_relevance_score", "descending")])
-
         return vector_results
 
     def rerank_fts(self, query: str, fts_results: pa.Table):
         fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
             fts_results = fts_results.drop_columns(["_score"])
-
         fts_results = fts_results.sort_by([("_relevance_score", "descending")])
-
         return fts_results
 
     @cached_property

python/python/lancedb/rerankers/voyageai.py

@@ -63,6 +63,9 @@ class VoyageAIReranker(Reranker):
         )
 
     def _rerank(self, result_set: pa.Table, query: str):
+        result_set = self._handle_empty_results(result_set)
+        if len(result_set) == 0:
+            return result_set
         docs = result_set[self.column].to_pylist()
         response = self._client.rerank(
             query=query,
@@ -101,24 +104,14 @@ class VoyageAIReranker(Reranker):
             )
         return combined_results
 
-    def rerank_vector(
-        self,
-        query: str,
-        vector_results: pa.Table,
-    ):
-        result_set = self._rerank(vector_results, query)
+    def rerank_vector(self, query: str, vector_results: pa.Table):
+        vector_results = self._rerank(vector_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_distance"])
+            vector_results = vector_results.drop_columns(["_distance"])
+        return vector_results
 
-        return result_set
-
-    def rerank_fts(
-        self,
-        query: str,
-        fts_results: pa.Table,
-    ):
-        result_set = self._rerank(fts_results, query)
+    def rerank_fts(self, query: str, fts_results: pa.Table):
+        fts_results = self._rerank(fts_results, query)
         if self.score == "relevance":
-            result_set = result_set.drop_columns(["_score"])
-
-        return result_set
+            fts_results = fts_results.drop_columns(["_score"])
+        return fts_results

python/python/lancedb/table.py

@@ -52,6 +52,7 @@ from .query import (
     AsyncHybridQuery,
     AsyncQuery,
     AsyncVectorQuery,
+    FullTextQuery,
     LanceEmptyQueryBuilder,
     LanceFtsQueryBuilder,
     LanceHybridQueryBuilder,
@@ -76,6 +77,7 @@ if TYPE_CHECKING:
         OptimizeStats,
         CleanupStats,
         CompactionStats,
+        Tag,
     )
     from .db import LanceDBConnection
     from .index import IndexConfig
@@ -581,6 +583,35 @@ class Table(ABC):
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def tags(self) -> Tags:
+        """Tag management for the table.
+
+        Similar to Git, tags are a way to add metadata to a specific version of the
+        table.
+
+        .. warning::
+
+            Tagged versions are exempted from the :py:meth:`cleanup_old_versions()`
+            process.
+
+            To remove a version that has been tagged, you must first
+            :py:meth:`~Tags.delete` the associated tag.
+
+        Examples
+        --------
+
+        .. code-block:: python
+
+            table = db.open_table("my_table")
+            table.tags.create("v2-prod-20250203", 10)
+
+            tags = table.tags.list()
+
+        """
+        raise NotImplementedError
+
     @property
     @abstractmethod
     def embedding_functions(self) -> Dict[str, EmbeddingFunctionConfig]:
@@ -630,6 +661,7 @@ class Table(ABC):
         index_cache_size: Optional[int] = None,
         *,
         index_type: VectorIndexType = "IVF_PQ",
+        wait_timeout: Optional[timedelta] = None,
         num_bits: int = 8,
         max_iterations: int = 50,
         sample_rate: int = 256,
@@ -665,6 +697,8 @@ class Table(ABC):
         num_bits: int
             The number of bits to encode sub-vectors. Only used with the IVF_PQ index.
             Only 4 and 8 are supported.
+        wait_timeout: timedelta, optional
+            The timeout to wait if indexing is asynchronous.
         """
         raise NotImplementedError
 
@@ -688,6 +722,30 @@ class Table(ABC):
         """
         raise NotImplementedError
 
+    def wait_for_index(
+        self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)
+    ) -> None:
+        """
+        Wait for indexing to complete for the given index names.
+        This will poll the table until all the indices are fully indexed,
+        or raise a timeout exception if the timeout is reached.
+
+        Parameters
+        ----------
+        index_names: str
+            The name of the indices to poll
+        timeout: timedelta
+            Timeout to wait for asynchronous indexing. The default is 5 minutes.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def stats(self) -> TableStatistics:
+        """
+        Retrieve table and fragment statistics.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def create_scalar_index(
         self,
@@ -695,6 +753,7 @@ class Table(ABC):
         *,
         replace: bool = True,
         index_type: ScalarIndexType = "BTREE",
+        wait_timeout: Optional[timedelta] = None,
     ):
         """Create a scalar index on a column.
 
@@ -707,7 +766,8 @@ class Table(ABC):
             Replace the existing index if it exists.
         index_type: Literal["BTREE", "BITMAP", "LABEL_LIST"], default "BTREE"
             The type of index to create.
-
+        wait_timeout: timedelta, optional
+            The timeout to wait if indexing is asynchronous.
         Examples
         --------
 
@@ -766,6 +826,7 @@ class Table(ABC):
         stem: bool = False,
         remove_stop_words: bool = False,
         ascii_folding: bool = False,
+        wait_timeout: Optional[timedelta] = None,
     ):
         """Create a full-text search index on the table.
 
@@ -821,6 +882,8 @@ class Table(ABC):
         ascii_folding : bool, default False
             Whether to fold ASCII characters. This converts accented characters to
             their ASCII equivalent. For example, "café" would be converted to "cafe".
+        wait_timeout: timedelta, optional
+            The timeout to wait if indexing is asynchronous.
         """
         raise NotImplementedError
 
@@ -899,10 +962,12 @@ class Table(ABC):
         >>> table = db.create_table("my_table", data)
         >>> new_data = pa.table({"a": [2, 3, 4], "b": ["x", "y", "z"]})
         >>> # Perform a "upsert" operation
-        >>> table.merge_insert("a")             \\
+        >>> stats = table.merge_insert("a")     \\
         ...      .when_matched_update_all()     \\
         ...      .when_not_matched_insert_all() \\
         ...      .execute(new_data)
+        >>> stats
+        {'num_inserted_rows': 1, 'num_updated_rows': 2, 'num_deleted_rows': 0}
         >>> # The order of new rows is non-deterministic since we use
         >>> # a hash-join as part of this operation and so we sort here
         >>> table.to_arrow().sort_by("a").to_pandas()
@@ -919,7 +984,9 @@ class Table(ABC):
     @abstractmethod
     def search(
         self,
-        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        query: Optional[
+            Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
+        ] = None,
         vector_column_name: Optional[str] = None,
         query_type: QueryType = "auto",
         ordering_field_name: Optional[str] = None,
@@ -1004,9 +1071,19 @@ class Table(ABC):
 
     @abstractmethod
     def _execute_query(
-        self, query: Query, batch_size: Optional[int] = None
+        self,
+        query: Query,
+        *,
+        batch_size: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
     ) -> pa.RecordBatchReader: ...
 
+    @abstractmethod
+    def _explain_plan(self, query: Query, verbose: Optional[bool] = False) -> str: ...
+
+    @abstractmethod
+    def _analyze_plan(self, query: Query) -> str: ...
+
     @abstractmethod
     def _do_merge(
         self,
@@ -1262,16 +1339,21 @@ class Table(ABC):
         """
 
     @abstractmethod
-    def add_columns(self, transforms: Dict[str, str]):
+    def add_columns(
+        self, transforms: Dict[str, str] | pa.Field | List[pa.Field] | pa.Schema
+    ):
         """
         Add new columns with defined values.
 
         Parameters
         ----------
-        transforms: Dict[str, str]
+        transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema
             A map of column name to a SQL expression to use to calculate the
             value of the new column. These expressions will be evaluated for
             each row in the table, and can reference existing columns.
+            Alternatively, a pyarrow Field or Schema can be provided to add
+            new columns with the specified data types. The new columns will
+            be initialized with null values.
         """
 
     @abstractmethod
@@ -1311,7 +1393,7 @@ class Table(ABC):
         """
 
     @abstractmethod
-    def checkout(self, version: int):
+    def checkout(self, version: Union[int, str]):
         """
         Checks out a specific version of the Table
 
@@ -1326,6 +1408,12 @@ class Table(ABC):
         Any operation that modifies the table will fail while the table is in a checked
         out state.
 
+        Parameters
+        ----------
+        version: int | str,
+            The version to check out. A version number (`int`) or a tag
+            (`str`) can be provided.
+
         To return the table to a normal state use `[Self::checkout_latest]`
         """
 
@@ -1339,6 +1427,21 @@ class Table(ABC):
         It can also be used to undo a `[Self::checkout]` operation
         """
 
+    @abstractmethod
+    def restore(self, version: Optional[int] = None):
+        """Restore a version of the table. This is an in-place operation.
+
+        This creates a new version where the data is equivalent to the
+        specified previous version. Data is not copied (as of python-v0.2.1).
+
+        Parameters
+        ----------
+        version : int, default None
+            The version to restore. If unspecified then restores the currently
+            checked out version. If the currently checked out version is the
+            latest version then this is a no-op.
+        """
+
     @abstractmethod
     def list_versions(self) -> List[Dict[str, Any]]:
         """List all versions of the table"""
@@ -1480,7 +1583,45 @@ class LanceTable(Table):
         """Get the current version of the table"""
         return LOOP.run(self._table.version())
 
-    def checkout(self, version: int):
+    @property
+    def tags(self) -> Tags:
+        """Tag management for the table.
+
+        Similar to Git, tags are a way to add metadata to a specific version of the
+        table.
+
+        .. warning::
+
+            Tagged versions are exempted from the :py:meth:`cleanup_old_versions()`
+            process.
+
+            To remove a version that has been tagged, you must first
+            :py:meth:`~Tags.delete` the associated tag.
+
+        Returns
+        -------
+        Tags
+            The tag manager for managing tags for the table.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table",
+        ...    [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table.tags.create("v1", table.version)
+        >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
+        >>> tags = table.tags.list()
+        >>> print(tags["v1"]["version"])
+        1
+        >>> table.checkout("v1")
+        >>> table.to_pandas()
+               vector    type
+        0  [1.1, 0.9]  vector
+        """
+        return Tags(self._table)
+
+    def checkout(self, version: Union[int, str]):
         """Checkout a version of the table. This is an in-place operation.
 
         This allows viewing previous versions of the table. If you wish to
@@ -1492,8 +1633,9 @@ class LanceTable(Table):
 
         Parameters
         ----------
-        version : int
-            The version to checkout.
+        version: int | str,
+            The version to check out. A version number (`int`) or a tag
+            (`str`) can be provided.
 
         Examples
         --------
@@ -1712,8 +1854,40 @@ class LanceTable(Table):
         )
 
     def drop_index(self, name: str) -> None:
+        """
+        Drops an index from the table
+
+        Parameters
+        ----------
+        name: str
+            The name of the index to drop
+        """
         return LOOP.run(self._table.drop_index(name))
 
+    def prewarm_index(self, name: str) -> None:
+        """
+        Prewarms an index in the table
+
+        This loads the entire index into memory
+
+        If the index does not fit into the available cache this call
+        may be wasteful
+
+        Parameters
+        ----------
+        name: str
+            The name of the index to prewarm
+        """
+        return LOOP.run(self._table.prewarm_index(name))
+
+    def wait_for_index(
+        self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)
+    ) -> None:
+        return LOOP.run(self._table.wait_for_index(index_names, timeout))
+
+    def stats(self) -> TableStatistics:
+        return LOOP.run(self._table.stats())
+
     def create_scalar_index(
         self,
         column: str,
@@ -2013,7 +2187,9 @@ class LanceTable(Table):
     @overload
     def search(
         self,
-        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        query: Optional[
+            Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
+        ] = None,
         vector_column_name: Optional[str] = None,
         query_type: Literal["hybrid"] = "hybrid",
         ordering_field_name: Optional[str] = None,
@@ -2032,7 +2208,9 @@ class LanceTable(Table):
 
     def search(
         self,
-        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        query: Optional[
+            Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
+        ] = None,
         vector_column_name: Optional[str] = None,
         query_type: QueryType = "auto",
         ordering_field_name: Optional[str] = None,
@@ -2104,6 +2282,8 @@ class LanceTable(Table):
             and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
+        if isinstance(query, FullTextQuery):
+            query_type = "fts"
         vector_column_name = infer_vector_column_name(
             schema=self.schema,
             query_type=query_type,
@@ -2279,9 +2459,15 @@ class LanceTable(Table):
         LOOP.run(self._table.update(values, where=where, updates_sql=values_sql))
 
     def _execute_query(
-        self, query: Query, batch_size: Optional[int] = None
+        self,
+        query: Query,
+        *,
+        batch_size: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
     ) -> pa.RecordBatchReader:
-        async_iter = LOOP.run(self._table._execute_query(query, batch_size))
+        async_iter = LOOP.run(
+            self._table._execute_query(query, batch_size=batch_size, timeout=timeout)
+        )
 
         def iter_sync():
             try:
@@ -2292,8 +2478,11 @@ class LanceTable(Table):
 
         return pa.RecordBatchReader.from_batches(async_iter.schema, iter_sync())
 
-    def _explain_plan(self, query: Query) -> str:
-        return LOOP.run(self._table._explain_plan(query))
+    def _explain_plan(self, query: Query, verbose: Optional[bool] = False) -> str:
+        return LOOP.run(self._table._explain_plan(query, verbose))
+
+    def _analyze_plan(self, query: Query) -> str:
+        return LOOP.run(self._table._analyze_plan(query))
 
     def _do_merge(
         self,
@@ -2302,7 +2491,9 @@ class LanceTable(Table):
         on_bad_vectors: OnBadVectorsType,
         fill_value: float,
     ):
-        LOOP.run(self._table._do_merge(merge, new_data, on_bad_vectors, fill_value))
+        return LOOP.run(
+            self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
+        )
 
     @deprecation.deprecated(
         deprecated_in="0.21.0",
@@ -2442,7 +2633,9 @@ class LanceTable(Table):
         """
         return LOOP.run(self._table.index_stats(index_name))
 
-    def add_columns(self, transforms: Dict[str, str]):
+    def add_columns(
+        self, transforms: Dict[str, str] | pa.field | List[pa.field] | pa.Schema
+    ):
         LOOP.run(self._table.add_columns(transforms))
 
     def alter_columns(self, *alterations: Iterable[Dict[str, str]]):
@@ -2890,6 +3083,7 @@ class AsyncTable:
         config: Optional[
             Union[IvfFlat, IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS]
         ] = None,
+        wait_timeout: Optional[timedelta] = None,
     ):
         """Create an index to speed up queries
 
@@ -2914,6 +3108,8 @@ class AsyncTable:
             For advanced configuration you can specify the type of index you would
             like to create.   You can also specify index-specific parameters when
             creating an index object.
+        wait_timeout: timedelta, optional
+            The timeout to wait if indexing is asynchronous.
         """
         if config is not None:
             if not isinstance(
@@ -2924,7 +3120,9 @@ class AsyncTable:
                     " Bitmap, LabelList, or FTS"
                 )
         try:
-            await self._inner.create_index(column, index=config, replace=replace)
+            await self._inner.create_index(
+                column, index=config, replace=replace, wait_timeout=wait_timeout
+            )
         except ValueError as e:
             if "not support the requested language" in str(e):
                 supported_langs = ", ".join(lang_mapping.values())
@@ -2952,6 +3150,46 @@ class AsyncTable:
         """
         await self._inner.drop_index(name)
 
+    async def prewarm_index(self, name: str) -> None:
+        """
+        Prewarm an index in the table.
+
+        Parameters
+        ----------
+        name: str
+            The name of the index to prewarm
+
+        Notes
+        -----
+        This will load the index into memory.  This may reduce the cold-start time for
+        future queries.  If the index does not fit in the cache then this call may be
+        wasteful.
+        """
+        await self._inner.prewarm_index(name)
+
+    async def wait_for_index(
+        self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)
+    ) -> None:
+        """
+        Wait for indexing to complete for the given index names.
+        This will poll the table until all the indices are fully indexed,
+        or raise a timeout exception if the timeout is reached.
+
+        Parameters
+        ----------
+        index_names: str
+            The name of the indices to poll
+        timeout: timedelta
+            Timeout to wait for asynchronous indexing. The default is 5 minutes.
+        """
+        await self._inner.wait_for_index(index_names, timeout)
+
+    async def stats(self) -> TableStatistics:
+        """
+        Retrieve table and fragment statistics.
+        """
+        return await self._inner.stats()
+
     async def add(
         self,
         data: DATA,
@@ -3043,10 +3281,12 @@ class AsyncTable:
         >>> table = db.create_table("my_table", data)
         >>> new_data = pa.table({"a": [2, 3, 4], "b": ["x", "y", "z"]})
         >>> # Perform a "upsert" operation
-        >>> table.merge_insert("a")             \\
+        >>> stats = table.merge_insert("a")     \\
         ...      .when_matched_update_all()     \\
         ...      .when_not_matched_insert_all() \\
         ...      .execute(new_data)
+        >>> stats
+        {'num_inserted_rows': 1, 'num_updated_rows': 2, 'num_deleted_rows': 0}
         >>> # The order of new rows is non-deterministic since we use
         >>> # a hash-join as part of this operation and so we sort here
         >>> table.to_arrow().sort_by("a").to_pandas()
@@ -3103,7 +3343,9 @@ class AsyncTable:
     @overload
     async def search(
         self,
-        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        query: Optional[
+            Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
+        ] = None,
         vector_column_name: Optional[str] = None,
         query_type: Literal["vector"] = ...,
         ordering_field_name: Optional[str] = None,
@@ -3112,7 +3354,9 @@ class AsyncTable:
 
     async def search(
         self,
-        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        query: Optional[
+            Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
+        ] = None,
         vector_column_name: Optional[str] = None,
         query_type: QueryType = "auto",
         ordering_field_name: Optional[str] = None,
@@ -3171,8 +3415,10 @@ class AsyncTable:
         async def get_embedding_func(
             vector_column_name: Optional[str],
             query_type: QueryType,
-            query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]],
+            query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]],
         ) -> Tuple[str, EmbeddingFunctionConfig]:
+            if isinstance(query, FullTextQuery):
+                query_type = "fts"
             schema = await self.schema()
             vector_column_name = infer_vector_column_name(
                 schema=schema,
@@ -3222,6 +3468,8 @@ class AsyncTable:
             if is_embedding(query):
                 vector_query = query
                 query_type = "vector"
+            elif isinstance(query, FullTextQuery):
+                query_type = "fts"
             elif isinstance(query, str):
                 try:
                     (
@@ -3342,13 +3590,15 @@ class AsyncTable:
             async_query = async_query.nearest_to_text(
                 query.full_text_query.query, query.full_text_query.columns
             )
-            if query.full_text_query.limit is not None:
-                async_query = async_query.limit(query.full_text_query.limit)
 
         return async_query
 
     async def _execute_query(
-        self, query: Query, batch_size: Optional[int] = None
+        self,
+        query: Query,
+        *,
+        batch_size: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
     ) -> pa.RecordBatchReader:
         # The sync table calls into this method, so we need to map the
         # query to the async version of the query and run that here. This is only
@@ -3356,12 +3606,19 @@ class AsyncTable:
 
         async_query = self._sync_query_to_async(query)
 
-        return await async_query.to_batches(max_batch_length=batch_size)
+        return await async_query.to_batches(
+            max_batch_length=batch_size, timeout=timeout
+        )
 
-    async def _explain_plan(self, query: Query) -> str:
+    async def _explain_plan(self, query: Query, verbose: Optional[bool]) -> str:
         # This method is used by the sync table
         async_query = self._sync_query_to_async(query)
-        return await async_query.explain_plan()
+        return await async_query.explain_plan(verbose)
+
+    async def _analyze_plan(self, query: Query) -> str:
+        # This method is used by the sync table
+        async_query = self._sync_query_to_async(query)
+        return await async_query.analyze_plan()
 
     async def _do_merge(
         self,
@@ -3385,7 +3642,7 @@ class AsyncTable:
         )
         if isinstance(data, pa.Table):
             data = pa.RecordBatchReader.from_batches(data.schema, data.to_batches())
-        await self._inner.execute_merge_insert(
+        return await self._inner.execute_merge_insert(
             data,
             dict(
                 on=merge._on,
@@ -3501,7 +3758,9 @@ class AsyncTable:
 
         return await self._inner.update(updates_sql, where)
 
-    async def add_columns(self, transforms: dict[str, str]):
+    async def add_columns(
+        self, transforms: dict[str, str] | pa.field | List[pa.field] | pa.Schema
+    ):
         """
         Add new columns with defined values.
 
@@ -3511,8 +3770,19 @@ class AsyncTable:
             A map of column name to a SQL expression to use to calculate the
             value of the new column. These expressions will be evaluated for
             each row in the table, and can reference existing columns.
+            Alternatively, you can pass a pyarrow field or schema to add
+            new columns with NULLs.
         """
-        await self._inner.add_columns(list(transforms.items()))
+        if isinstance(transforms, pa.Field):
+            transforms = [transforms]
+        if isinstance(transforms, list) and all(
+            {isinstance(f, pa.Field) for f in transforms}
+        ):
+            transforms = pa.schema(transforms)
+        if isinstance(transforms, pa.Schema):
+            await self._inner.add_columns_with_schema(transforms)
+        else:
+            await self._inner.add_columns(list(transforms.items()))
 
     async def alter_columns(self, *alterations: Iterable[dict[str, Any]]):
         """
@@ -3573,7 +3843,7 @@ class AsyncTable:
 
         return versions
 
-    async def checkout(self, version: int):
+    async def checkout(self, version: int | str):
         """
         Checks out a specific version of the Table
 
@@ -3588,6 +3858,12 @@ class AsyncTable:
         Any operation that modifies the table will fail while the table is in a checked
         out state.
 
+        Parameters
+        ----------
+        version: int | str,
+            The version to check out. A version number (`int`) or a tag
+            (`str`) can be provided.
+
         To return the table to a normal state use `[Self::checkout_latest]`
         """
         try:
@@ -3610,7 +3886,7 @@ class AsyncTable:
         """
         await self._inner.checkout_latest()
 
-    async def restore(self):
+    async def restore(self, version: Optional[int] = None):
         """
         Restore the table to the currently checked out version
 
@@ -3623,7 +3899,25 @@ class AsyncTable:
         Once the operation concludes the table will no longer be in a checked
         out state and the read_consistency_interval, if any, will apply.
         """
-        await self._inner.restore()
+        await self._inner.restore(version)
+
+    @property
+    def tags(self) -> AsyncTags:
+        """Tag management for the dataset.
+
+        Similar to Git, tags are a way to add metadata to a specific version of the
+        dataset.
+
+        .. warning::
+
+            Tagged versions are exempted from the
+            :py:meth:`optimize(cleanup_older_than)` process.
+
+            To remove a version that has been tagged, you must first
+            :py:meth:`~Tags.delete` the associated tag.
+
+        """
+        return AsyncTags(self._inner)
 
     async def optimize(
         self,
@@ -3794,3 +4088,217 @@ class IndexStatistics:
     # a dictionary instead of a class.
     def __getitem__(self, key):
         return getattr(self, key)
+
+
+@dataclass
+class TableStatistics:
+    """
+    Statistics about a table and fragments.
+
+    Attributes
+    ----------
+    total_bytes: int
+        The total number of bytes in the table.
+    num_rows: int
+        The total number of rows in the table.
+    num_indices: int
+        The total number of indices in the table.
+    fragment_stats: FragmentStatistics
+        Statistics about fragments in the table.
+    """
+
+    total_bytes: int
+    num_rows: int
+    num_indices: int
+    fragment_stats: FragmentStatistics
+
+
+@dataclass
+class FragmentStatistics:
+    """
+    Statistics about fragments.
+
+    Attributes
+    ----------
+    num_fragments: int
+        The total number of fragments in the table.
+    num_small_fragments: int
+        The total number of small fragments in the table.
+        Small fragments have low row counts and may need to be compacted.
+    lengths: FragmentSummaryStats
+        Statistics about the number of rows in the table fragments.
+    """
+
+    num_fragments: int
+    num_small_fragments: int
+    lengths: FragmentSummaryStats
+
+
+@dataclass
+class FragmentSummaryStats:
+    """
+    Statistics about fragments sizes
+
+    Attributes
+    ----------
+    min: int
+        The number of rows in the fragment with the fewest rows.
+    max: int
+        The number of rows in the fragment with the most rows.
+    mean: int
+        The mean number of rows in the fragments.
+    p25: int
+        The 25th percentile of number of rows in the fragments.
+    p50: int
+        The 50th percentile of number of rows in the fragments.
+    p75: int
+        The 75th percentile of number of rows in the fragments.
+    p99: int
+        The 99th percentile of number of rows in the fragments.
+    """
+
+    min: int
+    max: int
+    mean: int
+    p25: int
+    p50: int
+    p75: int
+    p99: int
+
+
+class Tags:
+    """
+    Table tag manager.
+    """
+
+    def __init__(self, table):
+        self._table = table
+
+    def list(self) -> Dict[str, Tag]:
+        """
+        List all table tags.
+
+        Returns
+        -------
+        dict[str, Tag]
+            A dictionary mapping tag names to version numbers.
+        """
+        return LOOP.run(self._table.tags.list())
+
+    def get_version(self, tag: str) -> int:
+        """
+        Get the version of a tag.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to get the version for.
+        """
+        return LOOP.run(self._table.tags.get_version(tag))
+
+    def create(self, tag: str, version: int) -> None:
+        """
+        Create a tag for a given table version.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to create. This name must be unique among all tag
+            names for the table.
+        version: int,
+            The table version to tag.
+        """
+        LOOP.run(self._table.tags.create(tag, version))
+
+    def delete(self, tag: str) -> None:
+        """
+        Delete tag from the table.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to delete.
+        """
+        LOOP.run(self._table.tags.delete(tag))
+
+    def update(self, tag: str, version: int) -> None:
+        """
+        Update tag to a new version.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to update.
+        version: int,
+            The new table version to tag.
+        """
+        LOOP.run(self._table.tags.update(tag, version))
+
+
+class AsyncTags:
+    """
+    Async table tag manager.
+    """
+
+    def __init__(self, table):
+        self._table = table
+
+    async def list(self) -> Dict[str, Tag]:
+        """
+        List all table tags.
+
+        Returns
+        -------
+        dict[str, Tag]
+            A dictionary mapping tag names to version numbers.
+        """
+        return await self._table.tags.list()
+
+    async def get_version(self, tag: str) -> int:
+        """
+        Get the version of a tag.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to get the version for.
+        """
+        return await self._table.tags.get_version(tag)
+
+    async def create(self, tag: str, version: int) -> None:
+        """
+        Create a tag for a given table version.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to create. This name must be unique among all tag
+            names for the table.
+        version: int,
+            The table version to tag.
+        """
+        await self._table.tags.create(tag, version)
+
+    async def delete(self, tag: str) -> None:
+        """
+        Delete tag from the table.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to delete.
+        """
+        await self._table.tags.delete(tag)
+
+    async def update(self, tag: str, version: int) -> None:
+        """
+        Update tag to a new version.
+
+        Parameters
+        ----------
+        tag: str,
+            The name of the tag to update.
+        version: int,
+            The new table version to tag.
+        """
+        await self._table.tags.update(tag, version)

python/python/lancedb/util.py

@@ -253,9 +253,14 @@ def infer_vector_column_name(
     query: Optional[Any],  # inferred later in query builder
     vector_column_name: Optional[str],
 ):
-    if (vector_column_name is None and query is not None and query_type != "fts") or (
-        vector_column_name is None and query_type == "hybrid"
-    ):
+    if vector_column_name is not None:
+        return vector_column_name
+
+    if query_type == "fts":
+        # FTS queries do not require a vector column
+        return None
+
+    if query is not None or query_type == "hybrid":
         try:
             vector_column_name = inf_vector_column_query(schema)
         except Exception as e:

python/python/tests/docs/test_guide_tables.py

@@ -562,7 +562,7 @@ async def test_table_async():
     async_db = await lancedb.connect_async(uri, read_consistency_interval=timedelta(0))
     async_tbl = await async_db.open_table("test_table_async")
     # --8<-- [end:table_async_strong_consistency]
-    # --8<-- [start:table_async_ventual_consistency]
+    # --8<-- [start:table_async_eventual_consistency]
     uri = "data/sample-lancedb"
     async_db = await lancedb.connect_async(
         uri, read_consistency_interval=timedelta(seconds=5)

python/python/tests/docs/test_merge_insert.py

@@ -18,15 +18,19 @@ def test_upsert(mem_db):
         {"id": 1, "name": "Bobby"},
         {"id": 2, "name": "Charlie"},
     ]
-    (
+    stats = (
         table.merge_insert("id")
         .when_matched_update_all()
         .when_not_matched_insert_all()
         .execute(new_users)
     )
     table.count_rows()  # 3
+    stats  # {'num_inserted_rows': 1, 'num_updated_rows': 1, 'num_deleted_rows': 0}
     # --8<-- [end:upsert_basic]
     assert table.count_rows() == 3
+    assert stats["num_inserted_rows"] == 1
+    assert stats["num_updated_rows"] == 1
+    assert stats["num_deleted_rows"] == 0
 
 
 @pytest.mark.asyncio
@@ -44,15 +48,19 @@ async def test_upsert_async(mem_db_async):
         {"id": 1, "name": "Bobby"},
         {"id": 2, "name": "Charlie"},
     ]
-    await (
+    stats = await (
         table.merge_insert("id")
         .when_matched_update_all()
         .when_not_matched_insert_all()
         .execute(new_users)
     )
     await table.count_rows()  # 3
+    stats  # {'num_inserted_rows': 1, 'num_updated_rows': 1, 'num_deleted_rows': 0}
     # --8<-- [end:upsert_basic_async]
     assert await table.count_rows() == 3
+    assert stats["num_inserted_rows"] == 1
+    assert stats["num_updated_rows"] == 1
+    assert stats["num_deleted_rows"] == 0
 
 
 def test_insert_if_not_exists(mem_db):
@@ -69,10 +77,16 @@ def test_insert_if_not_exists(mem_db):
         {"domain": "google.com", "name": "Google"},
         {"domain": "facebook.com", "name": "Facebook"},
     ]
-    (table.merge_insert("domain").when_not_matched_insert_all().execute(new_domains))
+    stats = (
+        table.merge_insert("domain").when_not_matched_insert_all().execute(new_domains)
+    )
     table.count_rows()  # 3
+    stats  # {'num_inserted_rows': 1, 'num_updated_rows': 0, 'num_deleted_rows': 0}
     # --8<-- [end:insert_if_not_exists]
     assert table.count_rows() == 3
+    assert stats["num_inserted_rows"] == 1
+    assert stats["num_updated_rows"] == 0
+    assert stats["num_deleted_rows"] == 0
 
 
 @pytest.mark.asyncio
@@ -90,12 +104,16 @@ async def test_insert_if_not_exists_async(mem_db_async):
         {"domain": "google.com", "name": "Google"},
         {"domain": "facebook.com", "name": "Facebook"},
     ]
-    await (
+    stats = await (
         table.merge_insert("domain").when_not_matched_insert_all().execute(new_domains)
     )
     await table.count_rows()  # 3
+    stats  # {'num_inserted_rows': 1, 'num_updated_rows': 0, 'num_deleted_rows': 0}
     # --8<-- [end:insert_if_not_exists_async]
     assert await table.count_rows() == 3
+    assert stats["num_inserted_rows"] == 1
+    assert stats["num_updated_rows"] == 0
+    assert stats["num_deleted_rows"] == 0
 
 
 def test_replace_range(mem_db):
@@ -113,7 +131,7 @@ def test_replace_range(mem_db):
     new_chunks = [
         {"doc_id": 1, "chunk_id": 0, "text": "Baz"},
     ]
-    (
+    stats = (
         table.merge_insert(["doc_id", "chunk_id"])
         .when_matched_update_all()
         .when_not_matched_insert_all()
@@ -121,8 +139,12 @@ def test_replace_range(mem_db):
         .execute(new_chunks)
     )
     table.count_rows("doc_id = 1")  # 1
+    stats  # {'num_inserted_rows': 0, 'num_updated_rows': 1, 'num_deleted_rows': 1}
     # --8<-- [end:replace_range]
     assert table.count_rows("doc_id = 1") == 1
+    assert stats["num_inserted_rows"] == 0
+    assert stats["num_updated_rows"] == 1
+    assert stats["num_deleted_rows"] == 1
 
 
 @pytest.mark.asyncio
@@ -141,7 +163,7 @@ async def test_replace_range_async(mem_db_async):
     new_chunks = [
         {"doc_id": 1, "chunk_id": 0, "text": "Baz"},
     ]
-    await (
+    stats = await (
         table.merge_insert(["doc_id", "chunk_id"])
         .when_matched_update_all()
         .when_not_matched_insert_all()
@@ -149,5 +171,9 @@ async def test_replace_range_async(mem_db_async):
         .execute(new_chunks)
     )
     await table.count_rows("doc_id = 1")  # 1
+    stats  # {'num_inserted_rows': 0, 'num_updated_rows': 1, 'num_deleted_rows': 1}
     # --8<-- [end:replace_range_async]
     assert await table.count_rows("doc_id = 1") == 1
+    assert stats["num_inserted_rows"] == 0
+    assert stats["num_updated_rows"] == 1
+    assert stats["num_deleted_rows"] == 1

python/python/tests/docs/test_search.py

@@ -6,7 +6,9 @@ import lancedb
 
 # --8<-- [end:import-lancedb]
 # --8<-- [start:import-numpy]
+from lancedb.query import BoostQuery, MatchQuery
 import numpy as np
+import pyarrow as pa
 
 # --8<-- [end:import-numpy]
 # --8<-- [start:import-datetime]
@@ -154,6 +156,84 @@ async def test_vector_search_async():
     # --8<-- [end:search_result_async_as_list]
 
 
+def test_fts_fuzzy_query():
+    uri = "data/fuzzy-example"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table_fts_fuzzy",
+        data=pa.table(
+            {
+                "text": [
+                    "fa",
+                    "fo",  # spellchecker:disable-line
+                    "fob",
+                    "focus",
+                    "foo",
+                    "food",
+                    "foul",
+                ]
+            }
+        ),
+        mode="overwrite",
+    )
+    table.create_fts_index("text", use_tantivy=False, replace=True)
+
+    results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
+    assert len(results) == 4
+    assert set(results["text"].to_list()) == {
+        "foo",
+        "fo",  # 1 deletion # spellchecker:disable-line
+        "fob",  # 1 substitution
+        "food",  # 1 insertion
+    }
+
+
+def test_fts_boost_query():
+    uri = "data/boost-example"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table_fts_boost",
+        data=pa.table(
+            {
+                "title": [
+                    "The Hidden Gems of Travel",
+                    "Exploring Nature's Wonders",
+                    "Cultural Treasures Unveiled",
+                    "The Nightlife Chronicles",
+                    "Scenic Escapes and Challenges",
+                ],
+                "desc": [
+                    "A vibrant city with occasional traffic jams.",
+                    "Beautiful landscapes but overpriced tourist spots.",
+                    "Rich cultural heritage but humid summers.",
+                    "Bustling nightlife but noisy streets.",
+                    "Scenic views but limited public transport options.",
+                ],
+            }
+        ),
+        mode="overwrite",
+    )
+    table.create_fts_index("desc", use_tantivy=False, replace=True)
+
+    results = table.search(
+        BoostQuery(
+            MatchQuery("beautiful, cultural, nightlife", "desc"),
+            MatchQuery("bad traffic jams, overpriced", "desc"),
+        ),
+    ).to_pandas()
+
+    # we will hit 3 results because the positive query has 3 hits
+    assert len(results) == 3
+    # the one containing "overpriced" will be negatively boosted,
+    # so it will be the last one
+    assert (
+        results["desc"].to_list()[2]
+        == "Beautiful landscapes but overpriced tourist spots."
+    )
+
+
 def test_fts_native():
     # --8<-- [start:basic_fts]
     uri = "data/sample-lancedb"

python/python/tests/test_embeddings_slow.py

@@ -11,7 +11,8 @@ import pandas as pd
 import pyarrow as pa
 import pytest
 from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
+from lancedb.pydantic import LanceModel, Vector, MultiVector
+import requests
 
 # These are integration tests for embedding functions.
 # They are slow because they require downloading models
@@ -516,3 +517,125 @@ def test_voyageai_embedding_function():
 
     tbl.add(df)
     assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_embedding_function():
+    voyageai = (
+        get_registry().get("voyageai").create(name="voyage-multimodal-3", max_retries=0)
+    )
+
+    class Images(LanceModel):
+        label: str
+        image_uri: str = voyageai.SourceField()  # image uri as the source
+        image_bytes: bytes = voyageai.SourceField()  # image bytes as the source
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()  # vector column
+        vec_from_bytes: Vector(voyageai.ndims()) = (
+            voyageai.VectorField()
+        )  # Another vector column
+
+    db = lancedb.connect("~/lancedb")
+    table = db.create_table("test", schema=Images, mode="overwrite")
+    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+    ]
+    # get each uri as bytes
+    image_bytes = [requests.get(uri).content for uri in uris]
+    table.add(
+        pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
+    )
+    assert len(table.to_pandas()["vector"][0]) == voyageai.ndims()
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_embedding_text_function():
+    voyageai = (
+        get_registry().get("voyageai").create(name="voyage-multimodal-3", max_retries=0)
+    )
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect("~/lancedb")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    importlib.util.find_spec("colpali_engine") is None,
+    reason="colpali_engine not installed",
+)
+def test_colpali(tmp_path):
+    import requests
+    from lancedb.pydantic import LanceModel
+
+    db = lancedb.connect(tmp_path)
+    registry = get_registry()
+    func = registry.get("colpali").create()
+
+    class MediaItems(LanceModel):
+        text: str
+        image_uri: str = func.SourceField()
+        image_bytes: bytes = func.SourceField()
+        image_vectors: MultiVector(func.ndims()) = (
+            func.VectorField()
+        )  # Multivector image embeddings
+
+    table = db.create_table("media", schema=MediaItems)
+
+    texts = [
+        "a cute cat playing with yarn",
+        "a puppy in a flower field",
+        "a red sports car on the highway",
+        "a vintage bicycle leaning against a wall",
+        "a plate of delicious pasta",
+        "fresh fruit salad in a bowl",
+    ]
+
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+    ]
+
+    # Get images as bytes
+    image_bytes = [requests.get(uri).content for uri in uris]
+
+    table.add(
+        pd.DataFrame({"text": texts, "image_uri": uris, "image_bytes": image_bytes})
+    )
+
+    # Test text-to-image search
+    image_results = (
+        table.search("fluffy companion", vector_column_name="image_vectors")
+        .limit(1)
+        .to_pydantic(MediaItems)[0]
+    )
+    assert "cat" in image_results.text.lower() or "puppy" in image_results.text.lower()
+
+    # Verify multivector dimensions
+    first_row = table.to_arrow().to_pylist()[0]
+    assert len(first_row["image_vectors"]) > 1, "Should have multiple image vectors"
+    assert len(first_row["image_vectors"][0]) == func.ndims(), (
+        "Vector dimension mismatch"
+    )

python/python/tests/test_fts.py

@@ -20,7 +20,9 @@ from unittest import mock
 import lancedb as ldb
 from lancedb.db import DBConnection
 from lancedb.index import FTS
+from lancedb.query import BoostQuery, MatchQuery, MultiMatchQuery, PhraseQuery
 import numpy as np
+import pyarrow as pa
 import pandas as pd
 import pytest
 from utils import exception_output
@@ -178,11 +180,47 @@ def test_search_fts(table, use_tantivy):
     results = table.search("puppy").select(["id", "text"]).to_list()
     assert len(results) == 10
 
+    if not use_tantivy:
+        # Test with a query
+        results = (
+            table.search(MatchQuery("puppy", "text"))
+            .select(["id", "text"])
+            .limit(5)
+            .to_list()
+        )
+        assert len(results) == 5
+
+        # Test boost query
+        results = (
+            table.search(
+                BoostQuery(
+                    MatchQuery("puppy", "text"),
+                    MatchQuery("runs", "text"),
+                )
+            )
+            .select(["id", "text"])
+            .limit(5)
+            .to_list()
+        )
+        assert len(results) == 5
+
+        # Test multi match query
+        table.create_fts_index("text2", use_tantivy=use_tantivy)
+        results = (
+            table.search(MultiMatchQuery("puppy", ["text", "text2"]))
+            .select(["id", "text"])
+            .limit(5)
+            .to_list()
+        )
+        assert len(results) == 5
+        assert len(results[0]) == 3  # id, text, _score
+
 
 @pytest.mark.asyncio
 async def test_fts_select_async(async_table):
     tbl = await async_table
     await tbl.create_index("text", config=FTS())
+    await tbl.create_index("text2", config=FTS())
     results = (
         await tbl.query()
         .nearest_to_text("puppy")
@@ -193,6 +231,54 @@ async def test_fts_select_async(async_table):
     assert len(results) == 5
     assert len(results[0]) == 3  # id, text, _score
 
+    # Test with FullTextQuery
+    results = (
+        await tbl.query()
+        .nearest_to_text(MatchQuery("puppy", "text"))
+        .select(["id", "text"])
+        .limit(5)
+        .to_list()
+    )
+    assert len(results) == 5
+    assert len(results[0]) == 3  # id, text, _score
+
+    # Test with BoostQuery
+    results = (
+        await tbl.query()
+        .nearest_to_text(
+            BoostQuery(
+                MatchQuery("puppy", "text"),
+                MatchQuery("runs", "text"),
+            )
+        )
+        .select(["id", "text"])
+        .limit(5)
+        .to_list()
+    )
+    assert len(results) == 5
+    assert len(results[0]) == 3  # id, text, _score
+
+    # Test with MultiMatchQuery
+    results = (
+        await tbl.query()
+        .nearest_to_text(MultiMatchQuery("puppy", ["text", "text2"]))
+        .select(["id", "text"])
+        .limit(5)
+        .to_list()
+    )
+    assert len(results) == 5
+    assert len(results[0]) == 3  # id, text, _score
+
+    # Test with search() API
+    results = (
+        await (await tbl.search(MatchQuery("puppy", "text")))
+        .select(["id", "text"])
+        .limit(5)
+        .to_list()
+    )
+    assert len(results) == 5
+    assert len(results[0]) == 3  # id, text, _score
+
 
 def test_search_fts_phrase_query(table):
     table.create_fts_index("text", use_tantivy=False, with_position=False)
@@ -207,6 +293,13 @@ def test_search_fts_phrase_query(table):
     assert len(results) > len(phrase_results)
     assert len(phrase_results) > 0
 
+    # Test with a query
+    phrase_results = (
+        table.search(PhraseQuery("puppy runs", "text")).limit(100).to_list()
+    )
+    assert len(results) > len(phrase_results)
+    assert len(phrase_results) > 0
+
 
 @pytest.mark.asyncio
 async def test_search_fts_phrase_query_async(async_table):
@@ -227,6 +320,16 @@ async def test_search_fts_phrase_query_async(async_table):
     assert len(results) > len(phrase_results)
     assert len(phrase_results) > 0
 
+    # Test with a query
+    phrase_results = (
+        await async_table.query()
+        .nearest_to_text(PhraseQuery("puppy runs", "text"))
+        .limit(100)
+        .to_list()
+    )
+    assert len(results) > len(phrase_results)
+    assert len(phrase_results) > 0
+
 
 def test_search_fts_specify_column(table):
     table.create_fts_index("text", use_tantivy=False)
@@ -524,3 +627,32 @@ def test_language(mem_db: DBConnection):
     # Stop words -> no results
     results = table.search("la", query_type="fts").limit(5).to_list()
     assert len(results) == 0
+
+
+def test_fts_on_list(mem_db: DBConnection):
+    data = pa.table(
+        {
+            "text": [
+                ["lance database", "the", "search"],
+                ["lance database"],
+                ["lance", "search"],
+                ["database", "search"],
+                ["unrelated", "doc"],
+            ],
+            "vector": [
+                [1.0, 2.0, 3.0],
+                [4.0, 5.0, 6.0],
+                [7.0, 8.0, 9.0],
+                [10.0, 11.0, 12.0],
+                [13.0, 14.0, 15.0],
+            ],
+        }
+    )
+    table = mem_db.create_table("test", data=data)
+    table.create_fts_index("text", use_tantivy=False)
+
+    res = table.search("lance").limit(5).to_list()
+    assert len(res) == 3
+
+    res = table.search(PhraseQuery("lance database", "text")).limit(5).to_list()
+    assert len(res) == 2

python/python/tests/test_hybrid_query.py

@@ -4,13 +4,32 @@
 import lancedb
 
 from lancedb.query import LanceHybridQueryBuilder
+from lancedb.rerankers.rrf import RRFReranker
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
 import pytest_asyncio
 
 from lancedb.index import FTS
-from lancedb.table import AsyncTable
+from lancedb.table import AsyncTable, Table
+
+
+@pytest.fixture
+def sync_table(tmpdir_factory) -> Table:
+    tmp_path = str(tmpdir_factory.mktemp("data"))
+    db = lancedb.connect(tmp_path)
+    data = pa.table(
+        {
+            "text": pa.array(["a", "b", "cat", "dog"]),
+            "vector": pa.array(
+                [[0.1, 0.1], [2, 2], [-0.1, -0.1], [0.5, -0.5]],
+                type=pa.list_(pa.float32(), list_size=2),
+            ),
+        }
+    )
+    table = db.create_table("test", data)
+    table.create_fts_index("text", with_position=False, use_tantivy=False)
+    return table
 
 
 @pytest_asyncio.fixture
@@ -102,6 +121,42 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable):
     assert texts.count("a") == 1
 
 
+def test_hybrid_query_distance_range(sync_table: Table):
+    reranker = RRFReranker(return_score="all")
+    result = (
+        sync_table.search(query_type="hybrid")
+        .vector([0.0, 0.4])
+        .text("cat and dog")
+        .distance_range(lower_bound=0.2, upper_bound=0.5)
+        .rerank(reranker)
+        .limit(2)
+        .to_arrow()
+    )
+    assert len(result) == 2
+    print(result)
+    for dist in result["_distance"]:
+        if dist.is_valid:
+            assert 0.2 <= dist.as_py() <= 0.5
+
+
+@pytest.mark.asyncio
+async def test_hybrid_query_distance_range_async(table: AsyncTable):
+    reranker = RRFReranker(return_score="all")
+    result = await (
+        table.query()
+        .nearest_to([0.0, 0.4])
+        .nearest_to_text("cat and dog")
+        .distance_range(lower_bound=0.2, upper_bound=0.5)
+        .rerank(reranker)
+        .limit(2)
+        .to_arrow()
+    )
+    assert len(result) == 2
+    for dist in result["_distance"]:
+        if dist.is_valid:
+            assert 0.2 <= dist.as_py() <= 0.5
+
+
 @pytest.mark.asyncio
 async def test_explain_plan(table: AsyncTable):
     plan = await (
@@ -114,6 +169,16 @@ async def test_explain_plan(table: AsyncTable):
     assert "LanceScan" in plan
 
 
+@pytest.mark.asyncio
+async def test_analyze_plan(table: AsyncTable):
+    res = await (
+        table.query().nearest_to_text("dog").nearest_to([0.1, 0.1]).analyze_plan()
+    )
+
+    assert "AnalyzeExec" in res
+    assert "metrics=" in res
+
+
 def test_normalize_scores():
     cases = [
         (pa.array([0.1, 0.4]), pa.array([0.0, 1.0])),

python/python/tests/test_index.py

@@ -8,7 +8,7 @@ import pyarrow as pa
 import pytest
 import pytest_asyncio
 from lancedb import AsyncConnection, AsyncTable, connect_async
-from lancedb.index import BTree, IvfFlat, IvfPq, Bitmap, LabelList, HnswPq, HnswSq
+from lancedb.index import BTree, IvfFlat, IvfPq, Bitmap, LabelList, HnswPq, HnswSq, FTS
 
 
 @pytest_asyncio.fixture
@@ -31,6 +31,7 @@ async def some_table(db_async):
         {
             "id": list(range(NROWS)),
             "vector": sample_fixed_size_list_array(NROWS, DIM),
+            "fsb": pa.array([bytes([i]) for i in range(NROWS)], pa.binary(1)),
             "tags": [
                 [f"tag{random.randint(0, 8)}" for _ in range(2)] for _ in range(NROWS)
             ],
@@ -85,6 +86,16 @@ async def test_create_scalar_index(some_table: AsyncTable):
     assert len(indices) == 0
 
 
+@pytest.mark.asyncio
+async def test_create_fixed_size_binary_index(some_table: AsyncTable):
+    await some_table.create_index("fsb", config=BTree())
+    indices = await some_table.list_indices()
+    assert str(indices) == '[Index(BTree, columns=["fsb"], name="fsb_idx")]'
+    assert len(indices) == 1
+    assert indices[0].index_type == "BTree"
+    assert indices[0].columns == ["fsb"]
+
+
 @pytest.mark.asyncio
 async def test_create_bitmap_index(some_table: AsyncTable):
     await some_table.create_index("id", config=Bitmap())
@@ -108,6 +119,18 @@ async def test_create_label_list_index(some_table: AsyncTable):
     assert str(indices) == '[Index(LabelList, columns=["tags"], name="tags_idx")]'
 
 
+@pytest.mark.asyncio
+async def test_full_text_search_index(some_table: AsyncTable):
+    await some_table.create_index("tags", config=FTS(with_position=False))
+    indices = await some_table.list_indices()
+    assert str(indices) == '[Index(FTS, columns=["tags"], name="tags_idx")]'
+
+    await some_table.prewarm_index("tags_idx")
+
+    res = await (await some_table.search("tag0")).to_arrow()
+    assert res.num_rows > 0
+
+
 @pytest.mark.asyncio
 async def test_create_vector_index(some_table: AsyncTable):
     # Can create

python/python/tests/test_pydantic.py

@@ -9,7 +9,13 @@ from typing import List, Optional, Tuple
 import pyarrow as pa
 import pydantic
 import pytest
-from lancedb.pydantic import PYDANTIC_VERSION, LanceModel, Vector, pydantic_to_schema
+from lancedb.pydantic import (
+    PYDANTIC_VERSION,
+    LanceModel,
+    Vector,
+    pydantic_to_schema,
+    MultiVector,
+)
 from pydantic import BaseModel
 from pydantic import Field
 
@@ -354,3 +360,55 @@ def test_optional_nested_model():
             ),
         ]
     )
+
+
+def test_multi_vector():
+    class TestModel(pydantic.BaseModel):
+        vec: MultiVector(8)
+
+    schema = pydantic_to_schema(TestModel)
+    assert schema == pa.schema(
+        [pa.field("vec", pa.list_(pa.list_(pa.float32(), 8)), True)]
+    )
+
+    with pytest.raises(pydantic.ValidationError):
+        TestModel(vec=[[1.0] * 7])
+
+    with pytest.raises(pydantic.ValidationError):
+        TestModel(vec=[[1.0] * 9])
+
+    TestModel(vec=[[1.0] * 8])
+    TestModel(vec=[[1.0] * 8, [2.0] * 8])
+
+    TestModel(vec=[])
+
+
+def test_multi_vector_nullable():
+    class NullableModel(pydantic.BaseModel):
+        vec: MultiVector(16, nullable=False)
+
+    schema = pydantic_to_schema(NullableModel)
+    assert schema == pa.schema(
+        [pa.field("vec", pa.list_(pa.list_(pa.float32(), 16)), False)]
+    )
+
+    class DefaultModel(pydantic.BaseModel):
+        vec: MultiVector(16)
+
+    schema = pydantic_to_schema(DefaultModel)
+    assert schema == pa.schema(
+        [pa.field("vec", pa.list_(pa.list_(pa.float32(), 16)), True)]
+    )
+
+
+def test_multi_vector_in_lance_model():
+    class TestModel(LanceModel):
+        id: int
+        vectors: MultiVector(16) = Field(default=[[0.0] * 16])
+
+    schema = pydantic_to_schema(TestModel)
+    assert schema == TestModel.to_arrow_schema()
+    assert TestModel.field_names() == ["id", "vectors"]
+
+    t = TestModel(id=1)
+    assert t.vectors == [[0.0] * 16]

python/python/tests/test_query.py

@@ -257,7 +257,9 @@ async def test_distance_range_with_new_rows_async():
         }
     )
     table = await conn.create_table("test", data)
-    table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2))
+    await table.create_index(
+        "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2)
+    )
 
     q = [0, 0]
     rs = await table.query().nearest_to(q).to_arrow()
@@ -511,7 +513,8 @@ def test_query_builder_with_different_vector_column():
             columns=["b"],
             vector_column="foo_vector",
         ),
-        None,
+        batch_size=None,
+        timeout=None,
     )
 
 
@@ -702,6 +705,20 @@ async def test_fast_search_async(tmp_path):
     assert "LanceScan" not in plan
 
 
+def test_analyze_plan(table):
+    q = LanceVectorQueryBuilder(table, [0, 0], "vector")
+    res = q.analyze_plan()
+    assert "AnalyzeExec" in res
+    assert "metrics=" in res
+
+
+@pytest.mark.asyncio
+async def test_analyze_plan_async(table_async: AsyncTable):
+    res = await table_async.query().nearest_to(pa.array([1, 2])).analyze_plan()
+    assert "AnalyzeExec" in res
+    assert "metrics=" in res
+
+
 def test_explain_plan(table):
     q = LanceVectorQueryBuilder(table, [0, 0], "vector")
     plan = q.explain_plan(verbose=True)
@@ -1062,3 +1079,67 @@ async def test_query_serialization_async(table_async: AsyncTable):
         full_text_query=FullTextSearchQuery(columns=[], query="foo"),
         with_row_id=False,
     )
+
+
+def test_query_timeout(tmp_path):
+    # Use local directory instead of memory:// to add a bit of latency to
+    # operations so a timeout of zero will trigger exceptions.
+    db = lancedb.connect(tmp_path)
+    data = pa.table(
+        {
+            "text": ["a", "b"],
+            "vector": pa.FixedSizeListArray.from_arrays(
+                pc.random(4).cast(pa.float32()), 2
+            ),
+        }
+    )
+    table = db.create_table("test", data)
+    table.create_fts_index("text", use_tantivy=False)
+
+    with pytest.raises(Exception, match="Query timeout"):
+        table.search().where("text = 'a'").to_list(timeout=timedelta(0))
+
+    with pytest.raises(Exception, match="Query timeout"):
+        table.search([0.0, 0.0]).to_arrow(timeout=timedelta(0))
+
+    with pytest.raises(Exception, match="Query timeout"):
+        table.search("a", query_type="fts").to_pandas(timeout=timedelta(0))
+
+    with pytest.raises(Exception, match="Query timeout"):
+        table.search(query_type="hybrid").vector([0.0, 0.0]).text("a").to_arrow(
+            timeout=timedelta(0)
+        )
+
+
+@pytest.mark.asyncio
+async def test_query_timeout_async(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table(
+        {
+            "text": ["a", "b"],
+            "vector": pa.FixedSizeListArray.from_arrays(
+                pc.random(4).cast(pa.float32()), 2
+            ),
+        }
+    )
+    table = await db.create_table("test", data)
+    await table.create_index("text", config=FTS())
+
+    with pytest.raises(Exception, match="Query timeout"):
+        await table.query().where("text != 'a'").to_list(timeout=timedelta(0))
+
+    with pytest.raises(Exception, match="Query timeout"):
+        await table.vector_search([0.0, 0.0]).to_arrow(timeout=timedelta(0))
+
+    with pytest.raises(Exception, match="Query timeout"):
+        await (await table.search("a", query_type="fts")).to_pandas(
+            timeout=timedelta(0)
+        )
+
+    with pytest.raises(Exception, match="Query timeout"):
+        await (
+            table.query()
+            .nearest_to_text("a")
+            .nearest_to([0.0, 0.0])
+            .to_list(timeout=timedelta(0))
+        )

python/python/tests/test_remote_db.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
+import re
 from concurrent.futures import ThreadPoolExecutor
 import contextlib
 from datetime import timedelta
@@ -235,6 +235,10 @@ def test_table_add_in_threadpool():
 
 def test_table_create_indices():
     def handler(request):
+        index_stats = dict(
+            index_type="IVF_PQ", num_indexed_rows=1000, num_unindexed_rows=0
+        )
+
         if request.path == "/v1/table/test/create_index/":
             request.send_response(200)
             request.end_headers()
@@ -258,6 +262,47 @@ def test_table_create_indices():
                 )
             )
             request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/list/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(
+                dict(
+                    indexes=[
+                        {
+                            "index_name": "id_idx",
+                            "columns": ["id"],
+                        },
+                        {
+                            "index_name": "text_idx",
+                            "columns": ["text"],
+                        },
+                        {
+                            "index_name": "vector_idx",
+                            "columns": ["vector"],
+                        },
+                    ]
+                )
+            )
+            request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/id_idx/stats/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(index_stats)
+            request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/text_idx/stats/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(index_stats)
+            request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/vector_idx/stats/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(index_stats)
+            request.wfile.write(payload.encode())
         elif "/drop/" in request.path:
             request.send_response(200)
             request.end_headers()
@@ -269,14 +314,125 @@ def test_table_create_indices():
         # Parameters are well-tested through local and async tests.
         # This is a smoke-test.
         table = db.create_table("test", [{"id": 1}])
-        table.create_scalar_index("id")
-        table.create_fts_index("text")
-        table.create_scalar_index("vector")
+        table.create_scalar_index("id", wait_timeout=timedelta(seconds=2))
+        table.create_fts_index("text", wait_timeout=timedelta(seconds=2))
+        table.create_index(
+            vector_column_name="vector", wait_timeout=timedelta(seconds=10)
+        )
+        table.wait_for_index(["id_idx"], timedelta(seconds=2))
+        table.wait_for_index(["text_idx", "vector_idx"], timedelta(seconds=2))
         table.drop_index("vector_idx")
         table.drop_index("id_idx")
         table.drop_index("text_idx")
 
 
+def test_table_wait_for_index_timeout():
+    def handler(request):
+        index_stats = dict(
+            index_type="BTREE", num_indexed_rows=1000, num_unindexed_rows=1
+        )
+
+        if request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(
+                dict(
+                    version=1,
+                    schema=dict(
+                        fields=[
+                            dict(name="id", type={"type": "int64"}, nullable=False),
+                        ]
+                    ),
+                )
+            )
+            request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/list/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(
+                dict(
+                    indexes=[
+                        {
+                            "index_name": "id_idx",
+                            "columns": ["id"],
+                        },
+                    ]
+                )
+            )
+            request.wfile.write(payload.encode())
+        elif request.path == "/v1/table/test/index/id_idx/stats/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(index_stats)
+            print(f"{index_stats=}")
+            request.wfile.write(payload.encode())
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        with pytest.raises(
+            RuntimeError,
+            match=re.escape(
+                'Timeout error: timed out waiting for indices: ["id_idx"] after 1s'
+            ),
+        ):
+            table.wait_for_index(["id_idx"], timedelta(seconds=1))
+
+
+def test_stats():
+    stats = {
+        "total_bytes": 38,
+        "num_rows": 2,
+        "num_indices": 0,
+        "fragment_stats": {
+            "num_fragments": 1,
+            "num_small_fragments": 1,
+            "lengths": {
+                "min": 2,
+                "max": 2,
+                "mean": 2,
+                "p25": 2,
+                "p50": 2,
+                "p75": 2,
+                "p99": 2,
+            },
+        },
+    }
+
+    def handler(request):
+        if request.path == "/v1/table/test/create/?mode=create":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/stats/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            payload = json.dumps(stats)
+            request.wfile.write(payload.encode())
+        else:
+            print(request.path)
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}])
+        res = table.stats()
+        print(f"{res=}")
+        assert res == stats
+
+
 @contextlib.contextmanager
 def query_test_table(query_handler, *, server_version=Version("0.1.0")):
     def handler(request):
@@ -444,6 +600,16 @@ def test_query_sync_fts():
             "prefilter": True,
             "with_row_id": True,
             "version": None,
+        } or body == {
+            "full_text_query": {
+                "query": "puppy",
+                "columns": ["description", "name"],
+            },
+            "k": 42,
+            "vector": [],
+            "prefilter": True,
+            "with_row_id": True,
+            "version": None,
         }
 
         return pa.table({"id": [1, 2, 3]})

python/python/tests/test_rerankers.py

@@ -457,3 +457,45 @@ def test_voyageai_reranker(tmp_path, use_tantivy):
     reranker = VoyageAIReranker(model_name="rerank-2")
     table, schema = get_test_table(tmp_path, use_tantivy)
     _run_test_reranker(reranker, table, "single player experience", None, schema)
+
+
+def test_empty_result_reranker():
+    pytest.importorskip("sentence_transformers")
+    db = lancedb.connect("memory://")
+
+    # Define schema
+    schema = pa.schema(
+        [
+            ("id", pa.int64()),
+            ("text", pa.string()),
+            ("vector", pa.list_(pa.float32(), 128)),  # 128-dimensional vector
+        ]
+    )
+
+    # Create empty table with schema
+    empty_table = db.create_table("empty_table", schema=schema, mode="overwrite")
+    empty_table.create_fts_index("text", use_tantivy=False, replace=True)
+    for reranker in [
+        CrossEncoderReranker(),
+        # ColbertReranker(),
+        # AnswerdotaiRerankers(),
+        # OpenaiReranker(),
+        # JinaReranker(),
+        # VoyageAIReranker(model_name="rerank-2"),
+    ]:
+        results = (
+            empty_table.search(list(range(128)))
+            .limit(3)
+            .rerank(reranker, "query")
+            .to_arrow()
+        )
+        # check if empty set contains _relevance_score column
+        assert "_relevance_score" in results.column_names
+        assert len(results) == 0
+
+        results = (
+            empty_table.search("query", query_type="fts")
+            .limit(3)
+            .rerank(reranker)
+            .to_arrow()
+        )

python/python/tests/test_table.py

@@ -9,9 +9,9 @@ from typing import List
 from unittest.mock import patch
 
 import lancedb
+from lancedb.dependencies import _PANDAS_AVAILABLE
 from lancedb.index import HnswPq, HnswSq, IvfPq
 import numpy as np
-import pandas as pd
 import polars as pl
 import pyarrow as pa
 import pyarrow.dataset
@@ -138,13 +138,16 @@ def test_create_table(mem_db: DBConnection):
         {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
     ]
-    df = pd.DataFrame(rows)
-    pa_table = pa.Table.from_pandas(df, schema=schema)
+    pa_table = pa.Table.from_pylist(rows, schema=schema)
     data = [
         ("Rows", rows),
-        ("pd_DataFrame", df),
         ("pa_Table", pa_table),
     ]
+    if _PANDAS_AVAILABLE:
+        import pandas as pd
+
+        df = pd.DataFrame(rows)
+        data.append(("pd_DataFrame", df))
 
     for name, d in data:
         tbl = mem_db.create_table(name, data=d, schema=schema).to_arrow()
@@ -296,7 +299,7 @@ def test_add_subschema(mem_db: DBConnection):
 
     data = {"price": 10.0, "item": "foo"}
     table.add([data])
-    data = pd.DataFrame({"price": [2.0], "vector": [[3.1, 4.1]]})
+    data = pa.Table.from_pydict({"price": [2.0], "vector": [[3.1, 4.1]]})
     table.add(data)
     data = {"price": 3.0, "vector": [5.9, 26.5], "item": "bar"}
     table.add([data])
@@ -405,6 +408,7 @@ def test_add_nullability(mem_db: DBConnection):
 
 
 def test_add_pydantic_model(mem_db: DBConnection):
+    pytest.importorskip("pandas")
     # https://github.com/lancedb/lancedb/issues/562
 
     class Metadata(BaseModel):
@@ -473,10 +477,10 @@ def test_polars(mem_db: DBConnection):
     table = mem_db.create_table("test", data=pl.DataFrame(data))
     assert len(table) == 2
 
-    result = table.to_pandas()
-    assert np.allclose(result["vector"].tolist(), data["vector"])
-    assert result["item"].tolist() == data["item"]
-    assert np.allclose(result["price"].tolist(), data["price"])
+    result = table.to_arrow()
+    assert np.allclose(result["vector"].to_pylist(), data["vector"])
+    assert result["item"].to_pylist() == data["item"]
+    assert np.allclose(result["price"].to_pylist(), data["price"])
 
     schema = pa.schema(
         [
@@ -525,6 +529,113 @@ def test_versioning(mem_db: DBConnection):
     assert len(table) == 2
 
 
+def test_tags(mem_db: DBConnection):
+    table = mem_db.create_table(
+        "test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+
+    table.tags.create("tag1", 1)
+    tags = table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 1
+
+    table.add(
+        data=[
+            {"vector": [10.0, 11.0], "item": "baz", "price": 30.0},
+        ],
+    )
+
+    table.tags.create("tag2", 2)
+    tags = table.tags.list()
+    assert "tag1" in tags
+    assert "tag2" in tags
+    assert tags["tag1"]["version"] == 1
+    assert tags["tag2"]["version"] == 2
+
+    table.tags.delete("tag2")
+    table.tags.update("tag1", 2)
+    tags = table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 2
+
+    table.tags.update("tag1", 1)
+    tags = table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 1
+
+    table.checkout("tag1")
+    assert table.version == 1
+    assert table.count_rows() == 2
+    table.tags.create("tag2", 2)
+    table.checkout("tag2")
+    assert table.version == 2
+    assert table.count_rows() == 3
+    table.checkout_latest()
+    table.add(
+        data=[
+            {"vector": [12.0, 13.0], "item": "baz", "price": 40.0},
+        ],
+    )
+
+
+@pytest.mark.asyncio
+async def test_async_tags(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table(
+        "test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+
+    await table.tags.create("tag1", 1)
+    tags = await table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 1
+
+    await table.add(
+        data=[
+            {"vector": [10.0, 11.0], "item": "baz", "price": 30.0},
+        ],
+    )
+
+    await table.tags.create("tag2", 2)
+    tags = await table.tags.list()
+    assert "tag1" in tags
+    assert "tag2" in tags
+    assert tags["tag1"]["version"] == 1
+    assert tags["tag2"]["version"] == 2
+
+    await table.tags.delete("tag2")
+    await table.tags.update("tag1", 2)
+    tags = await table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 2
+
+    await table.tags.update("tag1", 1)
+    tags = await table.tags.list()
+    assert "tag1" in tags
+    assert tags["tag1"]["version"] == 1
+
+    await table.checkout("tag1")
+    assert await table.version() == 1
+    assert await table.count_rows() == 2
+    await table.tags.create("tag2", 2)
+    await table.checkout("tag2")
+    assert await table.version() == 2
+    assert await table.count_rows() == 3
+    await table.checkout_latest()
+    await table.add(
+        data=[
+            {"vector": [12.0, 13.0], "item": "baz", "price": 40.0},
+        ],
+    )
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_method(mock_create_index, mem_db: DBConnection):
     table = mem_db.create_table(
@@ -688,7 +799,7 @@ def test_delete(mem_db: DBConnection):
     assert len(table.list_versions()) == 2
     assert table.version == 2
     assert len(table) == 1
-    assert table.to_pandas()["id"].tolist() == [1]
+    assert table.to_arrow()["id"].to_pylist() == [1]
 
 
 def test_update(mem_db: DBConnection):
@@ -852,6 +963,7 @@ def test_merge_insert(mem_db: DBConnection):
     ids=["pa.Table", "pd.DataFrame", "rows"],
 )
 def test_merge_insert_subschema(mem_db: DBConnection, data_format):
+    pytest.importorskip("pandas")
     initial_data = pa.table(
         {"id": range(3), "a": [1.0, 2.0, 3.0], "c": ["x", "x", "x"]}
     )
@@ -948,7 +1060,7 @@ def test_create_with_embedding_function(mem_db: DBConnection):
 
     func = MockTextEmbeddingFunction.create()
     texts = ["hello world", "goodbye world", "foo bar baz fizz buzz"]
-    df = pd.DataFrame({"text": texts, "vector": func.compute_source_embeddings(texts)})
+    df = pa.table({"text": texts, "vector": func.compute_source_embeddings(texts)})
 
     conf = EmbeddingFunctionConfig(
         source_column="text", vector_column="vector", function=func
@@ -973,7 +1085,7 @@ def test_create_f16_table(mem_db: DBConnection):
         text: str
         vector: Vector(32, value_type=pa.float16())
 
-    df = pd.DataFrame(
+    df = pa.table(
         {
             "text": [f"s-{i}" for i in range(512)],
             "vector": [np.random.randn(32).astype(np.float16) for _ in range(512)],
@@ -986,7 +1098,7 @@ def test_create_f16_table(mem_db: DBConnection):
     table.add(df)
     table.create_index(num_partitions=2, num_sub_vectors=2)
 
-    query = df.vector.iloc[2]
+    query = df["vector"][2].as_py()
     expected = table.search(query).limit(2).to_arrow()
 
     assert "s-2" in expected["text"].to_pylist()
@@ -1002,7 +1114,7 @@ def test_add_with_embedding_function(mem_db: DBConnection):
     table = mem_db.create_table("my_table", schema=MyTable)
 
     texts = ["hello world", "goodbye world", "foo bar baz fizz buzz"]
-    df = pd.DataFrame({"text": texts})
+    df = pa.table({"text": texts})
     table.add(df)
 
     texts = ["the quick brown fox", "jumped over the lazy dog"]
@@ -1033,14 +1145,14 @@ def test_multiple_vector_columns(mem_db: DBConnection):
         {"vector1": v1, "vector2": v2, "text": "foo"},
         {"vector1": v2, "vector2": v1, "text": "bar"},
     ]
-    df = pd.DataFrame(data)
+    df = pa.Table.from_pylist(data)
     table.add(df)
 
     q = np.random.randn(10)
-    result1 = table.search(q, vector_column_name="vector1").limit(1).to_pandas()
-    result2 = table.search(q, vector_column_name="vector2").limit(1).to_pandas()
+    result1 = table.search(q, vector_column_name="vector1").limit(1).to_arrow()
+    result2 = table.search(q, vector_column_name="vector2").limit(1).to_arrow()
 
-    assert result1["text"].iloc[0] != result2["text"].iloc[0]
+    assert result1["text"][0] != result2["text"][0]
 
 
 def test_create_scalar_index(mem_db: DBConnection):
@@ -1078,22 +1190,22 @@ def test_empty_query(mem_db: DBConnection):
         "my_table",
         data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
     )
-    df = table.search().select(["id"]).where("text='bar'").limit(1).to_pandas()
-    val = df.id.iloc[0]
+    df = table.search().select(["id"]).where("text='bar'").limit(1).to_arrow()
+    val = df["id"][0].as_py()
     assert val == 1
 
     table = mem_db.create_table("my_table2", data=[{"id": i} for i in range(100)])
-    df = table.search().select(["id"]).to_pandas()
-    assert len(df) == 100
+    df = table.search().select(["id"]).to_arrow()
+    assert df.num_rows == 100
     # None is the same as default
-    df = table.search().select(["id"]).limit(None).to_pandas()
-    assert len(df) == 100
+    df = table.search().select(["id"]).limit(None).to_arrow()
+    assert df.num_rows == 100
     # invalid limist is the same as None, wihch is the same as default
-    df = table.search().select(["id"]).limit(-1).to_pandas()
-    assert len(df) == 100
+    df = table.search().select(["id"]).limit(-1).to_arrow()
+    assert df.num_rows == 100
     # valid limit should work
-    df = table.search().select(["id"]).limit(42).to_pandas()
-    assert len(df) == 42
+    df = table.search().select(["id"]).limit(42).to_arrow()
+    assert df.num_rows == 42
 
 
 def test_search_with_schema_inf_single_vector(mem_db: DBConnection):
@@ -1112,14 +1224,14 @@ def test_search_with_schema_inf_single_vector(mem_db: DBConnection):
         {"vector_col": v1, "text": "foo"},
         {"vector_col": v2, "text": "bar"},
     ]
-    df = pd.DataFrame(data)
+    df = pa.Table.from_pylist(data)
     table.add(df)
 
     q = np.random.randn(10)
-    result1 = table.search(q, vector_column_name="vector_col").limit(1).to_pandas()
-    result2 = table.search(q).limit(1).to_pandas()
+    result1 = table.search(q, vector_column_name="vector_col").limit(1).to_arrow()
+    result2 = table.search(q).limit(1).to_arrow()
 
-    assert result1["text"].iloc[0] == result2["text"].iloc[0]
+    assert result1["text"][0].as_py() == result2["text"][0].as_py()
 
 
 def test_search_with_schema_inf_multiple_vector(mem_db: DBConnection):
@@ -1139,12 +1251,12 @@ def test_search_with_schema_inf_multiple_vector(mem_db: DBConnection):
         {"vector1": v1, "vector2": v2, "text": "foo"},
         {"vector1": v2, "vector2": v1, "text": "bar"},
     ]
-    df = pd.DataFrame(data)
+    df = pa.Table.from_pylist(data)
     table.add(df)
 
     q = np.random.randn(10)
     with pytest.raises(ValueError):
-        table.search(q).limit(1).to_pandas()
+        table.search(q).limit(1).to_arrow()
 
 
 def test_compact_cleanup(tmp_db: DBConnection):
@@ -1384,6 +1496,37 @@ async def test_add_columns_async(mem_db_async: AsyncConnection):
     assert data["new_col"].to_pylist() == [2, 3]
 
 
+@pytest.mark.asyncio
+async def test_add_columns_with_schema(mem_db_async: AsyncConnection):
+    data = pa.table({"id": [0, 1]})
+    table = await mem_db_async.create_table("my_table", data=data)
+    await table.add_columns(
+        [pa.field("x", pa.int64()), pa.field("vector", pa.list_(pa.float32(), 8))]
+    )
+
+    assert await table.schema() == pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("x", pa.int64()),
+            pa.field("vector", pa.list_(pa.float32(), 8)),
+        ]
+    )
+
+    table = await mem_db_async.create_table("table2", data=data)
+    await table.add_columns(
+        pa.schema(
+            [pa.field("y", pa.int64()), pa.field("emb", pa.list_(pa.float32(), 8))]
+        )
+    )
+    assert await table.schema() == pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("y", pa.int64()),
+            pa.field("emb", pa.list_(pa.float32(), 8)),
+        ]
+    )
+
+
 def test_alter_columns(mem_db: DBConnection):
     data = pa.table({"id": [0, 1]})
     table = mem_db.create_table("my_table", data=data)
@@ -1552,3 +1695,31 @@ def test_replace_field_metadata(tmp_path):
     schema = table.schema
     field = schema[0].metadata
     assert field == {b"foo": b"bar"}
+
+
+def test_stats(mem_db: DBConnection):
+    table = mem_db.create_table(
+        "my_table",
+        data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
+    )
+    assert len(table) == 2
+    stats = table.stats()
+    print(f"{stats=}")
+    assert stats == {
+        "total_bytes": 38,
+        "num_rows": 2,
+        "num_indices": 0,
+        "fragment_stats": {
+            "num_fragments": 1,
+            "num_small_fragments": 1,
+            "lengths": {
+                "min": 2,
+                "max": 2,
+                "mean": 2,
+                "p25": 2,
+                "p50": 2,
+                "p75": 2,
+                "p99": 2,
+            },
+        },
+    }

python/src/query.rs

@@ -2,25 +2,26 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use arrow::array::make_array;
 use arrow::array::Array;
 use arrow::array::ArrayData;
 use arrow::pyarrow::FromPyArrow;
 use arrow::pyarrow::IntoPyArrow;
-use lancedb::index::scalar::FullTextSearchQuery;
+use lancedb::index::scalar::{FtsQuery, FullTextSearchQuery, MatchQuery, PhraseQuery};
 use lancedb::query::QueryExecutionOptions;
 use lancedb::query::QueryFilter;
 use lancedb::query::{
     ExecutableQuery, Query as LanceDbQuery, QueryBase, Select, VectorQuery as LanceDbVectorQuery,
 };
 use lancedb::table::AnyQuery;
-use pyo3::exceptions::PyNotImplementedError;
 use pyo3::exceptions::PyRuntimeError;
+use pyo3::exceptions::{PyNotImplementedError, PyValueError};
 use pyo3::prelude::{PyAnyMethods, PyDictMethods};
 use pyo3::pymethods;
-use pyo3::types::PyDict;
 use pyo3::types::PyList;
+use pyo3::types::{PyDict, PyString};
 use pyo3::Bound;
 use pyo3::IntoPyObject;
 use pyo3::PyAny;
@@ -31,7 +32,7 @@ use pyo3_async_runtimes::tokio::future_into_py;
 
 use crate::arrow::RecordBatchStream;
 use crate::error::PythonErrorExt;
-use crate::util::parse_distance_type;
+use crate::util::{parse_distance_type, parse_fts_query};
 
 // Python representation of full text search parameters
 #[derive(Clone)]
@@ -45,9 +46,9 @@ pub struct PyFullTextSearchQuery {
 
 impl From<FullTextSearchQuery> for PyFullTextSearchQuery {
     fn from(query: FullTextSearchQuery) -> Self {
-        PyFullTextSearchQuery {
-            columns: query.columns,
-            query: query.query,
+        Self {
+            columns: query.columns().into_iter().collect(),
+            query: query.query.query().to_owned(),
             limit: query.limit,
             wand_factor: query.wand_factor,
         }
@@ -99,7 +100,7 @@ pub struct PyQueryRequest {
 impl From<AnyQuery> for PyQueryRequest {
     fn from(query: AnyQuery) -> Self {
         match query {
-            AnyQuery::Query(query_request) => PyQueryRequest {
+            AnyQuery::Query(query_request) => Self {
                 limit: query_request.limit,
                 offset: query_request.offset,
                 filter: query_request.filter.map(PyQueryFilter),
@@ -121,7 +122,7 @@ impl From<AnyQuery> for PyQueryRequest {
                 postfilter: None,
                 norm: None,
             },
-            AnyQuery::VectorQuery(vector_query) => PyQueryRequest {
+            AnyQuery::VectorQuery(vector_query) => Self {
                 limit: vector_query.base.limit,
                 offset: vector_query.base.offset,
                 filter: vector_query.base.filter.map(PyQueryFilter),
@@ -236,29 +237,69 @@ impl Query {
     }
 
     pub fn nearest_to_text(&mut self, query: Bound<'_, PyDict>) -> PyResult<FTSQuery> {
-        let query_text = query
+        let fts_query = query
             .get_item("query")?
             .ok_or(PyErr::new::<PyRuntimeError, _>(
                 "Query text is required for nearest_to_text",
-            ))?
-            .extract::<String>()?;
-        let columns = query
-            .get_item("columns")?
-            .map(|columns| columns.extract::<Vec<String>>())
-            .transpose()?;
+            ))?;
 
-        let fts_query = FullTextSearchQuery::new(query_text).columns(columns);
+        let query = if let Ok(query_text) = fts_query.downcast::<PyString>() {
+            let mut query_text = query_text.to_string();
+            let columns = query
+                .get_item("columns")?
+                .map(|columns| columns.extract::<Vec<String>>())
+                .transpose()?;
+
+            let is_phrase =
+                query_text.len() >= 2 && query_text.starts_with('"') && query_text.ends_with('"');
+            let is_multi_match = columns.as_ref().map(|cols| cols.len() > 1).unwrap_or(false);
+
+            if is_phrase {
+                // Remove the surrounding quotes for phrase queries
+                query_text = query_text[1..query_text.len() - 1].to_string();
+            }
+
+            let query: FtsQuery = match (is_phrase, is_multi_match) {
+                (false, _) => MatchQuery::new(query_text).into(),
+                (true, false) => PhraseQuery::new(query_text).into(),
+                (true, true) => {
+                    return Err(PyValueError::new_err(
+                        "Phrase queries cannot be used with multiple columns.",
+                    ));
+                }
+            };
+            let mut query = FullTextSearchQuery::new_query(query);
+            if let Some(cols) = columns {
+                if !cols.is_empty() {
+                    query = query.with_columns(&cols).map_err(|e| {
+                        PyValueError::new_err(format!(
+                            "Failed to set full text search columns: {}",
+                            e
+                        ))
+                    })?;
+                }
+            }
+            query
+        } else if let Ok(query) = fts_query.downcast::<PyDict>() {
+            let query = parse_fts_query(query)?;
+            FullTextSearchQuery::new_query(query)
+        } else {
+            return Err(PyValueError::new_err(
+                "query must be a string or a Query object",
+            ));
+        };
 
         Ok(FTSQuery {
-            fts_query,
             inner: self.inner.clone(),
+            fts_query: query,
         })
     }
 
-    #[pyo3(signature = (max_batch_length=None))]
+    #[pyo3(signature = (max_batch_length=None, timeout=None))]
     pub fn execute(
         self_: PyRef<'_, Self>,
         max_batch_length: Option<u32>,
+        timeout: Option<Duration>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
@@ -266,12 +307,15 @@ impl Query {
             if let Some(max_batch_length) = max_batch_length {
                 opts.max_batch_length = max_batch_length;
             }
+            if let Some(timeout) = timeout {
+                opts.timeout = Some(timeout);
+            }
             let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
             Ok(RecordBatchStream::new(inner_stream))
         })
     }
 
-    fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
+    pub fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
             inner
@@ -281,6 +325,16 @@ impl Query {
         })
     }
 
+    pub fn analyze_plan(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .analyze_plan()
+                .await
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+        })
+    }
+
     pub fn to_query_request(&self) -> PyQueryRequest {
         PyQueryRequest::from(AnyQuery::Query(self.inner.clone().into_request()))
     }
@@ -327,10 +381,11 @@ impl FTSQuery {
         self.inner = self.inner.clone().postfilter();
     }
 
-    #[pyo3(signature = (max_batch_length=None))]
+    #[pyo3(signature = (max_batch_length=None, timeout=None))]
     pub fn execute(
         self_: PyRef<'_, Self>,
         max_batch_length: Option<u32>,
+        timeout: Option<Duration>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_
             .inner
@@ -342,6 +397,9 @@ impl FTSQuery {
             if let Some(max_batch_length) = max_batch_length {
                 opts.max_batch_length = max_batch_length;
             }
+            if let Some(timeout) = timeout {
+                opts.timeout = Some(timeout);
+            }
             let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
             Ok(RecordBatchStream::new(inner_stream))
         })
@@ -365,8 +423,18 @@ impl FTSQuery {
         })
     }
 
+    pub fn analyze_plan(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .analyze_plan()
+                .await
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+        })
+    }
+
     pub fn get_query(&self) -> String {
-        self.fts_query.query.clone()
+        self.fts_query.query.query().to_owned()
     }
 
     pub fn to_query_request(&self) -> PyQueryRequest {
@@ -454,10 +522,11 @@ impl VectorQuery {
         self.inner = self.inner.clone().bypass_vector_index()
     }
 
-    #[pyo3(signature = (max_batch_length=None))]
+    #[pyo3(signature = (max_batch_length=None, timeout=None))]
     pub fn execute(
         self_: PyRef<'_, Self>,
         max_batch_length: Option<u32>,
+        timeout: Option<Duration>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
@@ -465,12 +534,15 @@ impl VectorQuery {
             if let Some(max_batch_length) = max_batch_length {
                 opts.max_batch_length = max_batch_length;
             }
+            if let Some(timeout) = timeout {
+                opts.timeout = Some(timeout);
+            }
             let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
             Ok(RecordBatchStream::new(inner_stream))
         })
     }
 
-    fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
+    pub fn explain_plan(self_: PyRef<'_, Self>, verbose: bool) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
             inner
@@ -480,6 +552,16 @@ impl VectorQuery {
         })
     }
 
+    pub fn analyze_plan(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            inner
+                .analyze_plan()
+                .await
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))
+        })
+    }
+
     pub fn nearest_to_text(&mut self, query: Bound<'_, PyDict>) -> PyResult<HybridQuery> {
         let base_query = self.inner.clone().into_plain();
         let fts_query = Query::new(base_query).nearest_to_text(query)?;
@@ -570,6 +652,11 @@ impl HybridQuery {
         self.inner_vec.bypass_vector_index();
     }
 
+    #[pyo3(signature = (lower_bound=None, upper_bound=None))]
+    pub fn distance_range(&mut self, lower_bound: Option<f32>, upper_bound: Option<f32>) {
+        self.inner_vec.distance_range(lower_bound, upper_bound);
+    }
+
     pub fn to_vector_query(&mut self) -> PyResult<VectorQuery> {
         Ok(VectorQuery {
             inner: self.inner_vec.inner.clone(),

python/src/table.rs

@@ -1,28 +1,28 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
-use arrow::{
-    datatypes::DataType,
-    ffi_stream::ArrowArrayStreamReader,
-    pyarrow::{FromPyArrow, ToPyArrow},
-};
-use lancedb::table::{
-    AddDataMode, ColumnAlteration, Duration, NewColumnTransform, OptimizeAction, OptimizeOptions,
-    Table as LanceDbTable,
-};
-use pyo3::{
-    exceptions::{PyKeyError, PyRuntimeError, PyValueError},
-    pyclass, pymethods,
-    types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods},
-    Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
-};
-use pyo3_async_runtimes::tokio::future_into_py;
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
 
 use crate::{
     error::PythonErrorExt,
     index::{extract_index_params, IndexConfig},
     query::Query,
 };
+use arrow::{
+    datatypes::{DataType, Schema},
+    ffi_stream::ArrowArrayStreamReader,
+    pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
+};
+use lancedb::table::{
+    AddDataMode, ColumnAlteration, Duration, NewColumnTransform, OptimizeAction, OptimizeOptions,
+    Table as LanceDbTable,
+};
+use pyo3::{
+    exceptions::{PyIOError, PyKeyError, PyRuntimeError, PyValueError},
+    pyclass, pymethods,
+    types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods, PyInt, PyString},
+    Bound, FromPyObject, PyAny, PyObject, PyRef, PyResult, Python,
+};
+use pyo3_async_runtimes::tokio::future_into_py;
 
 /// Statistics about a compaction operation.
 #[pyclass(get_all)]
@@ -176,15 +176,19 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (column, index=None, replace=None))]
+    #[pyo3(signature = (column, index=None, replace=None, wait_timeout=None))]
     pub fn create_index<'a>(
         self_: PyRef<'a, Self>,
         column: String,
         index: Option<Bound<'_, PyAny>>,
         replace: Option<bool>,
+        wait_timeout: Option<Bound<'_, PyAny>>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let index = extract_index_params(&index)?;
-        let mut op = self_.inner_ref()?.create_index(&[column], index);
+        let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
+        let mut op = self_
+            .inner_ref()?
+            .create_index_with_timeout(&[column], index, timeout);
         if let Some(replace) = replace {
             op = op.replace(replace);
         }
@@ -203,6 +207,34 @@ impl Table {
         })
     }
 
+    pub fn wait_for_index<'a>(
+        self_: PyRef<'a, Self>,
+        index_names: Vec<String>,
+        timeout: Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'a, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        let timeout = timeout.extract::<std::time::Duration>()?;
+        future_into_py(self_.py(), async move {
+            let index_refs = index_names
+                .iter()
+                .map(String::as_str)
+                .collect::<Vec<&str>>();
+            inner
+                .wait_for_index(&index_refs, timeout)
+                .await
+                .infer_error()?;
+            Ok(())
+        })
+    }
+
+    pub fn prewarm_index(self_: PyRef<'_, Self>, index_name: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.prewarm_index(&index_name).await.infer_error()?;
+            Ok(())
+        })
+    }
+
     pub fn list_indices(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
@@ -247,6 +279,40 @@ impl Table {
         })
     }
 
+    pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let stats = inner.stats().await.infer_error()?;
+            Python::with_gil(|py| {
+                let dict = PyDict::new(py);
+                dict.set_item("total_bytes", stats.total_bytes)?;
+                dict.set_item("num_rows", stats.num_rows)?;
+                dict.set_item("num_indices", stats.num_indices)?;
+
+                let fragment_stats = PyDict::new(py);
+                fragment_stats.set_item("num_fragments", stats.fragment_stats.num_fragments)?;
+                fragment_stats.set_item(
+                    "num_small_fragments",
+                    stats.fragment_stats.num_small_fragments,
+                )?;
+
+                let fragment_lengths = PyDict::new(py);
+                fragment_lengths.set_item("min", stats.fragment_stats.lengths.min)?;
+                fragment_lengths.set_item("max", stats.fragment_stats.lengths.max)?;
+                fragment_lengths.set_item("mean", stats.fragment_stats.lengths.mean)?;
+                fragment_lengths.set_item("p25", stats.fragment_stats.lengths.p25)?;
+                fragment_lengths.set_item("p50", stats.fragment_stats.lengths.p50)?;
+                fragment_lengths.set_item("p75", stats.fragment_stats.lengths.p75)?;
+                fragment_lengths.set_item("p99", stats.fragment_stats.lengths.p99)?;
+
+                fragment_stats.set_item("lengths", fragment_lengths)?;
+                dict.set_item("fragment_stats", fragment_stats)?;
+
+                Ok(Some(dict.unbind()))
+            })
+        })
+    }
+
     pub fn __repr__(&self) -> String {
         match &self.inner {
             None => format!("ClosedTable({})", self.name),
@@ -289,10 +355,26 @@ impl Table {
         })
     }
 
-    pub fn checkout(self_: PyRef<'_, Self>, version: u64) -> PyResult<Bound<'_, PyAny>> {
+    pub fn checkout(self_: PyRef<'_, Self>, version: PyObject) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move {
-            inner.checkout(version).await.infer_error()
+        let py = self_.py();
+        let (is_int, int_value, string_value) = if let Ok(i) = version.downcast_bound::<PyInt>(py) {
+            let num: u64 = i.extract()?;
+            (true, num, String::new())
+        } else if let Ok(s) = version.downcast_bound::<PyString>(py) {
+            let str_value = s.to_string();
+            (false, 0, str_value)
+        } else {
+            return Err(PyIOError::new_err(
+                "version must be an integer or a string.",
+            ));
+        };
+        future_into_py(py, async move {
+            if is_int {
+                inner.checkout(int_value).await.infer_error()
+            } else {
+                inner.checkout_tag(&string_value).await.infer_error()
+            }
         })
     }
 
@@ -303,18 +385,27 @@ impl Table {
         })
     }
 
-    pub fn restore(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    #[pyo3(signature = (version=None))]
+    pub fn restore(self_: PyRef<'_, Self>, version: Option<u64>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
-        future_into_py(
-            self_.py(),
-            async move { inner.restore().await.infer_error() },
-        )
+
+        future_into_py(self_.py(), async move {
+            if let Some(version) = version {
+                inner.checkout(version).await.infer_error()?;
+            }
+            inner.restore().await.infer_error()
+        })
     }
 
     pub fn query(&self) -> Query {
         Query::new(self.inner_ref().unwrap().query())
     }
 
+    #[getter]
+    pub fn tags(&self) -> PyResult<Tags> {
+        Ok(Tags::new(self.inner_ref()?.clone()))
+    }
+
     /// Optimize the on-disk data by compacting and pruning old data, for better performance.
     #[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None, retrain=None))]
     pub fn optimize(
@@ -398,8 +489,14 @@ impl Table {
         }
 
         future_into_py(self_.py(), async move {
-            builder.execute(Box::new(batches)).await.infer_error()?;
-            Ok(())
+            let stats = builder.execute(Box::new(batches)).await.infer_error()?;
+            Python::with_gil(|py| {
+                let dict = PyDict::new(py);
+                dict.set_item("num_inserted_rows", stats.num_inserted_rows)?;
+                dict.set_item("num_updated_rows", stats.num_updated_rows)?;
+                dict.set_item("num_deleted_rows", stats.num_deleted_rows)?;
+                Ok(dict.unbind())
+            })
         })
     }
 
@@ -440,6 +537,20 @@ impl Table {
         })
     }
 
+    pub fn add_columns_with_schema(
+        self_: PyRef<'_, Self>,
+        schema: PyArrowType<Schema>,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let arrow_schema = &schema.0;
+        let transform = NewColumnTransform::AllNulls(Arc::new(arrow_schema.clone()));
+
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.add_columns(transform, None).await.infer_error()?;
+            Ok(())
+        })
+    }
+
     pub fn alter_columns<'a>(
         self_: PyRef<'a, Self>,
         alterations: Vec<Bound<PyDict>>,
@@ -535,3 +646,72 @@ pub struct MergeInsertParams {
     when_not_matched_by_source_delete: bool,
     when_not_matched_by_source_condition: Option<String>,
 }
+
+#[pyclass]
+pub struct Tags {
+    inner: LanceDbTable,
+}
+
+impl Tags {
+    pub fn new(table: LanceDbTable) -> Self {
+        Self { inner: table }
+    }
+}
+
+#[pymethods]
+impl Tags {
+    pub fn list(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let tags = inner.tags().await.infer_error()?;
+            let res = tags.list().await.infer_error()?;
+
+            Python::with_gil(|py| {
+                let py_dict = PyDict::new(py);
+                for (key, contents) in res {
+                    let value_dict = PyDict::new(py);
+                    value_dict.set_item("version", contents.version)?;
+                    value_dict.set_item("manifest_size", contents.manifest_size)?;
+                    py_dict.set_item(key, value_dict)?;
+                }
+                Ok(py_dict.unbind())
+            })
+        })
+    }
+
+    pub fn get_version(self_: PyRef<'_, Self>, tag: String) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let tags = inner.tags().await.infer_error()?;
+            let res = tags.get_version(tag.as_str()).await.infer_error()?;
+            Ok(res)
+        })
+    }
+
+    pub fn create(self_: PyRef<Self>, tag: String, version: u64) -> PyResult<Bound<PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let mut tags = inner.tags().await.infer_error()?;
+            tags.create(tag.as_str(), version).await.infer_error()?;
+            Ok(())
+        })
+    }
+
+    pub fn delete(self_: PyRef<Self>, tag: String) -> PyResult<Bound<PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let mut tags = inner.tags().await.infer_error()?;
+            tags.delete(tag.as_str()).await.infer_error()?;
+            Ok(())
+        })
+    }
+
+    pub fn update(self_: PyRef<Self>, tag: String, version: u64) -> PyResult<Bound<PyAny>> {
+        let inner = self_.inner.clone();
+        future_into_py(self_.py(), async move {
+            let mut tags = inner.tags().await.infer_error()?;
+            tags.update(tag.as_str(), version).await.infer_error()?;
+            Ok(())
+        })
+    }
+}

python/src/util.rs

@@ -3,11 +3,15 @@
 
 use std::sync::Mutex;
 
+use lancedb::index::scalar::{BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, PhraseQuery};
 use lancedb::DistanceType;
+use pyo3::prelude::{PyAnyMethods, PyDictMethods, PyListMethods};
+use pyo3::types::PyDict;
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
     pyfunction, PyResult,
 };
+use pyo3::{Bound, PyAny};
 
 /// A wrapper around a rust builder
 ///
@@ -59,3 +63,117 @@ pub fn validate_table_name(table_name: &str) -> PyResult<()> {
     lancedb::utils::validate_table_name(table_name)
         .map_err(|e| PyValueError::new_err(e.to_string()))
 }
+
+pub fn parse_fts_query(query: &Bound<'_, PyDict>) -> PyResult<FtsQuery> {
+    let query_type = query.keys().get_item(0)?.extract::<String>()?;
+    let query_value = query
+        .get_item(&query_type)?
+        .ok_or(PyValueError::new_err(format!(
+            "Query type {} not found",
+            query_type
+        )))?;
+    let query_value = query_value.downcast::<PyDict>()?;
+
+    match query_type.as_str() {
+        "match" => {
+            let column = query_value.keys().get_item(0)?.extract::<String>()?;
+            let params = query_value
+                .get_item(&column)?
+                .ok_or(PyValueError::new_err(format!(
+                    "column {} not found",
+                    column
+                )))?;
+            let params = params.downcast::<PyDict>()?;
+
+            let query = params
+                .get_item("query")?
+                .ok_or(PyValueError::new_err("query not found"))?
+                .extract::<String>()?;
+            let boost = params
+                .get_item("boost")?
+                .ok_or(PyValueError::new_err("boost not found"))?
+                .extract::<f32>()?;
+            let fuzziness = params
+                .get_item("fuzziness")?
+                .ok_or(PyValueError::new_err("fuzziness not found"))?
+                .extract::<Option<u32>>()?;
+            let max_expansions = params
+                .get_item("max_expansions")?
+                .ok_or(PyValueError::new_err("max_expansions not found"))?
+                .extract::<usize>()?;
+
+            let query = MatchQuery::new(query)
+                .with_column(Some(column))
+                .with_boost(boost)
+                .with_fuzziness(fuzziness)
+                .with_max_expansions(max_expansions);
+            Ok(query.into())
+        }
+
+        "match_phrase" => {
+            let column = query_value.keys().get_item(0)?.extract::<String>()?;
+            let query = query_value
+                .get_item(&column)?
+                .ok_or(PyValueError::new_err(format!(
+                    "column {} not found",
+                    column
+                )))?
+                .extract::<String>()?;
+
+            let query = PhraseQuery::new(query).with_column(Some(column));
+            Ok(query.into())
+        }
+
+        "boost" => {
+            let positive: Bound<'_, PyAny> = query_value
+                .get_item("positive")?
+                .ok_or(PyValueError::new_err("positive not found"))?;
+            let positive = positive.downcast::<PyDict>()?;
+
+            let negative = query_value
+                .get_item("negative")?
+                .ok_or(PyValueError::new_err("negative not found"))?;
+            let negative = negative.downcast::<PyDict>()?;
+
+            let negative_boost = query_value
+                .get_item("negative_boost")?
+                .ok_or(PyValueError::new_err("negative_boost not found"))?
+                .extract::<f32>()?;
+
+            let positive_query = parse_fts_query(positive)?;
+            let negative_query = parse_fts_query(negative)?;
+            let query = BoostQuery::new(positive_query, negative_query, Some(negative_boost));
+
+            Ok(query.into())
+        }
+
+        "multi_match" => {
+            let query = query_value
+                .get_item("query")?
+                .ok_or(PyValueError::new_err("query not found"))?
+                .extract::<String>()?;
+
+            let columns = query_value
+                .get_item("columns")?
+                .ok_or(PyValueError::new_err("columns not found"))?
+                .extract::<Vec<String>>()?;
+
+            let boost = query_value
+                .get_item("boost")?
+                .ok_or(PyValueError::new_err("boost not found"))?
+                .extract::<Vec<f32>>()?;
+
+            let query = MultiMatchQuery::try_new(query, columns)
+                .and_then(|q| q.try_with_boosts(boost))
+                .map_err(|e| {
+                    PyValueError::new_err(format!("Error creating MultiMatchQuery: {}", e))
+                })?;
+            Ok(query.into())
+        }
+
+        _ => Err(PyValueError::new_err(format!(
+            "Unsupported query type: {}",
+            query_type
+        ))),
+    }
+}

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.18.2-beta.1"
+version = "0.19.1-beta.1"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true