Bump version: 0.6.0-beta.0 → 0.6.0

Bump version: 0.5.2 → 0.6.0-beta.0
Bump version: 0.9.0-beta.0 → 0.9.0
2025-12-23 05:19:58 +00:00 · 2024-06-25 00:25:45 +00:00 · 2024-06-25 00:25:45 +00:00 · 2024-06-25 00:25:27 +00:00 · 2024-06-25 00:25:27 +00:00 · 2024-06-24 17:22:57 -07:00
65 changed files with 2163 additions and 315 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.5.1"
+current_version = "0.6.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/.github/workflows/build_linux_wheel/action.yml
+++ b/.github/workflows/build_linux_wheel/action.yml
@@ -46,6 +46,7 @@ runs:
      with:
        command: build
        working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
        target: aarch64-unknown-linux-gnu
        manylinux: ${{ inputs.manylinux }}
        args: ${{ inputs.args }}
--- a/.github/workflows/build_mac_wheel/action.yml
+++ b/.github/workflows/build_mac_wheel/action.yml
@@ -21,5 +21,6 @@ runs:
      with:
        command: build
        args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
        working-directory: python
        interpreter: 3.${{ inputs.python-minor-version }}
--- a/.github/workflows/build_windows_wheel/action.yml
+++ b/.github/workflows/build_windows_wheel/action.yml
@@ -26,6 +26,7 @@ runs:
      with:
        command: build
        args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
        working-directory: python
    - uses: actions/upload-artifact@v3
      with:
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -3,7 +3,7 @@ name: NPM Publish
 on:
  push:
    tags:
-      - 'v*'
+      - "v*"

 jobs:
  node:
@@ -111,12 +111,11 @@ jobs:
            runner: ubuntu-latest
          - arch: aarch64
            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
+            runner: warp-ubuntu-latest-arm64-4x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      # To avoid OOM errors on ARM, we create a swap file.
      - name: Configure aarch64 build
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
@@ -323,7 +322,7 @@ jobs:
      - name: Publish to NPM
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        # By default, things are published to the latest tag. This is what is 
+        # By default, things are published to the latest tag. This is what is
        # installed by default if the user does not specify a version. This is
        # good for stable releases, but for pre-releases, we want to publish to
        # the "preview" tag so they can install with `npm install lancedb@preview`.
@@ -368,7 +367,7 @@ jobs:
      - uses: ./.github/workflows/update_package_lock_nodejs
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
-  
+
  gh-release:
    runs-on: ubuntu-latest
    permissions:
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -65,7 +65,7 @@ jobs:
          workspaces: python
      - name: Install
        run: |
-          pip install -e .[tests,dev,embeddings]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
          pip install tantivy
          pip install mlx
      - name: Doctest
@@ -189,7 +189,7 @@ jobs:
      - name: Install lancedb
        run: |
          pip install "pydantic<2"
-          pip install -e .[tests]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
          pip install tantivy
      - name: Run tests
        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
--- a/.github/workflows/run_tests/action.yml
+++ b/.github/workflows/run_tests/action.yml
@@ -15,7 +15,7 @@ runs:
    - name: Install lancedb
      shell: bash
      run: |
-        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
+        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
    - name: Setup localstack for integration tests
      if: ${{ inputs.integration == 'true' }}
      shell: bash
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
  hooks:
    - id: local-biome-check
      name: biome check
-      entry: npx @biomejs/biome check --config-path nodejs/biome.json nodejs/
+      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
      language: system
      types: [text]
      files: "nodejs/.*"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,11 +20,11 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]

 [workspace.dependencies]
-lance = { "version" = "=0.12.1", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.12.1" }
-lance-linalg = { "version" = "=0.12.1" }
-lance-testing = { "version" = "=0.12.1" }
-lance-datafusion = { "version" = "=0.12.1" }
+lance = { "version" = "=0.13.0", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.13.0" }
+lance-linalg = { "version" = "=0.13.0" }
+lance-testing = { "version" = "=0.13.0" }
+lance-datafusion = { "version" = "=0.13.0" }
 # Note that this one does not include pyarrow
 arrow = { version = "51.0", optional = false }
 arrow-array = "51.0"
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -106,6 +106,9 @@ nav:
          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
          - Configuring Storage: guides/storage.md
          - Sync -> Async Migration Guide: migration.md
+          - Tuning retrieval performance:
+              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+              - Reranking: guides/tuning_retrievers/2_reranking.md
      - 🧬 Managing embeddings:
          - Overview: embeddings/index.md
          - Embedding functions: embeddings/embedding_functions.md
@@ -121,7 +124,9 @@ nav:
          - LangChain:
            - LangChain 🔗: integrations/langchain.md
            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-          - LlamaIndex 🦙: https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
+          - LlamaIndex 🦙:
+            - LlamaIndex docs: integrations/llamaIndex.md
+            - LlamaIndex demo:  https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
          - Pydantic: python/pydantic.md
          - Voxel51: integrations/voxel51.md
          - PromptTools: integrations/prompttools.md
@@ -152,7 +157,7 @@ nav:
          - Overview: cloud/index.md
          - API reference:
              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/saas-modules.md
+              - 👾 JavaScript: javascript/modules.md

  - Quick start: basic.md
  - Concepts:
@@ -181,6 +186,9 @@ nav:
      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
      - Configuring Storage: guides/storage.md
      - Sync -> Async Migration Guide: migration.md
+      - Tuning retrieval performance:
+          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+          - Reranking: guides/tuning_retrievers/2_reranking.md
  - Managing Embeddings:
      - Overview: embeddings/index.md
      - Embedding functions: embeddings/embedding_functions.md
@@ -219,7 +227,7 @@ nav:
      - Overview: cloud/index.md
      - API reference:
          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/saas-modules.md
+          - 👾 JavaScript: javascript/modules.md

 extra_css:
  - styles/global.css
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -180,6 +180,9 @@ table.

 !!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."

+!!! info "Automatic embedding generation with Embedding API"
+    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
+
 ### Create an empty table

 Sometimes you may not have the data to insert into the table at creation time.
@@ -194,6 +197,9 @@ similar to a `CREATE TABLE` statement in SQL.
      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
      ```

+    !!! note "You can define schema in Pydantic"
+        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
+
 === "Typescript"

    ```typescript
@@ -424,6 +430,19 @@ Use the `drop_table()` method on the database to remove a table.
    })
    ```

+## Using the Embedding API
+You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
+    ```
+
+Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
+
+
 ## What's next

 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
--- a/docs/src/embeddings/default_embedding_functions.md
+++ b/docs/src/embeddings/default_embedding_functions.md
@@ -216,7 +216,7 @@ Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) py
 |------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
 | `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
 | `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`. |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
 | `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
 | `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |

@@ -365,6 +365,68 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```

+### Cohere Embeddings
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+```
+    * embed-english-v3.0
+    * embed-multilingual-v3.0
+    * embed-english-light-v3.0
+    * embed-multilingual-light-v3.0
+    * embed-english-v2.0
+    * embed-english-light-v2.0
+    * embed-multilingual-v2.0
+```
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+    
+    ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import EmbeddingFunctionRegistry
+
+        cohere = EmbeddingFunctionRegistry
+            .get_instance()
+            .get("cohere")
+            .create(name="embed-multilingual-v2.0")
+
+        class TextModel(LanceModel):
+            text: str = cohere.SourceField()
+            vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+        data = [ { "text": "hello world" },
+                { "text": "goodbye world" }]
+
+        db = lancedb.connect("~/.lancedb")
+        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+        tbl.add(data)
+    ```
+
 ### AWS Bedrock Text Embedding Functions
 AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
 You can do so by using `awscli` and also add your session_token:
--- a/docs/src/embeddings/embedding_functions.md
+++ b/docs/src/embeddings/embedding_functions.md
@@ -2,6 +2,9 @@ Representing multi-modal data as vector embeddings is becoming a standard practi

 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.

+!!! Note "LanceDB cloud doesn't support embedding functions yet"
+    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
+
 !!! warning
    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
--- a/docs/src/fts.md
+++ b/docs/src/fts.md
@@ -2,7 +2,6 @@

 LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)

-A hybrid search solution combining vector and full-text search is also on the way.

 ## Installation

@@ -55,6 +54,16 @@ This returns the result as a list of dictionaries as follows.
 !!! note
    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.

+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("text", tokenizer_name="en_stem")
+```
+
+The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+
 ## Index multiple columns

 If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
@@ -140,6 +149,7 @@ is treated as a phrase query.
 In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
 double quotes replaced by single quotes.

+
 ## Configurations

 By default, LanceDB configures a 1GB heap size limit for creating the index. You can
--- a/docs/src/guides/tables.md
+++ b/docs/src/guides/tables.md
@@ -452,6 +452,27 @@ After a table has been created, you can always add more data to it using the var
    tbl.add(pydantic_model_items)
    ```

+    ??? "Ingesting Pydantic models with LanceDB embedding API"
+        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
+
+        ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import get_registry
+
+        db = lancedb.connect("~/tmp")
+        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
+
+        class Schema(LanceModel):
+            text: str = embed_fcn.SourceField()
+            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
+
+        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
+        models = [Schema(text="hello"), Schema(text="world")]
+        tbl.add(models)
+        ```
+
+

 === "JavaScript"

@@ -636,6 +657,31 @@ The `values` parameter is used to provide the new values for the columns as lite

    When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.

+## Drop a table
+
+Use the `drop_table()` method on the database to remove a table.
+
+=== "Python"
+
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
+
+=== "Javascript/Typescript"
+
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised.
+
+
 ## Consistency

 In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
--- a/docs/src/guides/tuning_retrievers/1_query_types.md
+++ b/docs/src/guides/tuning_retrievers/1_query_types.md
@@ -0,0 +1,128 @@
+## Improving retriever performance
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+
+There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
+
+* Using different query types
+* Using hybrid search
+* Fine-tuning the embedding models
+* Using different embedding models
+
+Using different embedding models is something that's very specific to the use case and the data. So we will not discuss it here. In this section, we will discuss the first three techniques.
+
+
+!!! note "Note"
+    We'll be using a simple metric called "hit-rate" for evaluating the performance of the retriever across this guide. Hit-rate is the percentage of queries for which the retriever returned the correct answer in the top-k results. For example, if the retriever returned the correct answer in the top-3 results for 70% of the queries, then the hit-rate@3 is 0.7.
+
+
+## The dataset
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
+
+### Using different query types
+Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
+
+```python
+import lancedb
+import pandas as pd
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import Vector, LanceModel
+
+db = lancedb.connect("~/lancedb/query_types")
+df = pd.read_csv("data_qa.csv")
+
+embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.")
+
+class Schema(LanceModel):
+    context: str = embed_fcn.SourceField()
+    vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField()
+
+table = db.create_table("qa", schema=Schema)
+table.add(df[["context"]].to_dict(orient="records"))
+
+queries = df["query"].tolist()
+```
+
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
+
+* <b> Vector Search: </b>
+
+    ```python
+    table.search(quries[0], query_type="vector").limit(5).to_pandas()
+    ```
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
+
+    ```python
+    table.search(quries[0]).limit(5).to_pandas()
+    ```
+
+    Vector or semantic search is useful when you want to find documents that are similar to the query in terms of meaning.
+
+---
+
+* <b> Full-text Search: </b>
+    
+    FTS requires creating an index on the column you want to search on. `replace=True` will replace the existing index if it exists.
+    Once the index is created, you can search using the `fts` query type.
+    ```python
+    table.create_fts_index("context", replace=True)
+    table.search(quries[0], query_type="fts").limit(5).to_pandas()
+    ```
+
+    Full-text search is useful when you want to find documents that contain the query terms.
+
+---
+
+* <b> Hybrid Search: </b>
+
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
+    ```python
+    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
+    ```
+    Hybrid search requires a reranker to combine and rank the results from vector and full-text search. We'll cover reranking as a concept in the next section.
+
+    Hybrid search is useful when you want to combine the benefits of both vector and full-text search.
+
+    !!! note "Note"
+        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
+
+    
+
+### Hit rate evaluation results
+
+Now that we have seen how to run different query types on the dataset, let's evaluate the hit-rate of each query type on the dataset.
+For brevity, the entire evaluation script is not shown here. You can find the complete evaluation and benchmarking utility scripts [here](https://github.com/lancedb/ragged).
+
+Here are the hit-rate results for the dataset:
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Hybrid Search (w/ LinearCombinationReranker) | 0.645 |
+
+**Choosing query type** is very specific to the use case and the data. This synthetic dataset has been generated to be semantically challenging, i.e, the queries don't have a lot of keywords in common with the context. So, vector search performs better than full-text search. However, in real-world scenarios, full-text search might perform better than vector search. Hybrid search is a good choice when you want to combine the benefits of both vector and full-text search.
+
+### Evaluation results on other datasets
+
+The hit-rate results can vary based on the dataset and the query type. Here are the hit-rate results for the other datasets using the same embedding function.
+
+* <b> SQuAD Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.822 |
+    | Full-text Search | 0.835 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.8874 |
+
+* <b> Uber10K sec filing Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.608 |
+    | Full-text Search | 0.82 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.80 |
+
+In these standard datasets, FTS seems to perform much better than vector search because the queries have a lot of keywords in common with the context. So, in general choosing the query type is very specific to the use case and the data.
+
+
--- a/docs/src/guides/tuning_retrievers/2_reranking.md
+++ b/docs/src/guides/tuning_retrievers/2_reranking.md
@@ -0,0 +1,78 @@
+Continuing from the previous example, we can now rerank the results using more complex rerankers.
+
+## Reranking search results
+You can rerank any search results using a reranker. The syntax for reranking is as follows:
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker()
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+```
+Based on the `query_type`, the `rerank()` function can accept other arguments as well. For example, hybrid search accepts a `normalize` param to determine the score normalization method.
+
+!!! note "Note"
+    LanceDB provides a `Reranker` base class that can be extended to implement custom rerankers. Each reranker must implement the `rerank_hybrid` method. `rerank_vector` and `rerank_fts` methods are optional. For example, the `LinearCombinationReranker` only implements the `rerank_hybrid` method and so it can only be used for reranking hybrid search results.
+
+## Choosing a Reranker
+There are many rerankers available in LanceDB like `CrossEncoderReranker`, `CohereReranker`, and `ColBERT`. The choice of reranker depends on the dataset and the application. You can even implement you own custom reranker by extending the `Reranker` class. For more details about each available reranker and performance comparison, refer to the [rerankers](https://lancedb.github.io/lancedb/reranking/) documentation.
+
+In this example, we'll use the `CohereReranker` to rerank the search results. It requires  `cohere` to be installed and `COHERE_API_KEY` to be set in the environment. To get your API key, sign up on [Cohere](https://cohere.ai/).
+
+```python
+from lancedb.rerankers import CohereReranker
+
+# use Cohere reranker v3
+reranker = CohereReranker(model_name="rerank-english-v3.0") # default model is "rerank-english-v2.0"
+```
+
+### Reranking search results
+Now we can rerank all query type results using the `CohereReranker`:
+
+```python
+
+# rerank hybrid search results
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank vector search results
+table.search(quries[0], query_type="vector").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank fts search results
+table.search(quries[0], query_type="fts").rerank(reranker=reranker).limit(5).to_pandas()
+```
+
+Each reranker can accept additional arguments. For example, `CohereReranker` accepts `top_k` and `batch_size` params to control the number of documents to rerank and the batch size for reranking respectively. Similarly, a custom reranker can accept any number of arguments based on the implementation. For example, a reranker can accept a `filter` that implements some custom logic to filter out documents before reranking.
+
+## Results
+
+Let us take a look at the same datasets from the previous sections, using the same embedding table but with Cohere reranker applied to all query types.
+
+!!! note "Note"
+    When reranking fts or vector search results, the search results are over-fetched by a factor of 2 and then reranked. From the reranked set, `top_k` (5 in this case) results are taken. This is done because reranking will have no effect on the hit-rate if we only fetch the `top_k` results.
+
+### Synthetic LLama2 paper dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.640 |
+| FTS   |  0.595  |
+| Reranked vector | 0.677    |
+| Reranked fts  | 0.672    |
+| Hybrid | 0.759 |
+
+### SQuAD Dataset
+
+
+### Uber10K sec filing Dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.608 |
+| FTS   |  0.824  |
+| Reranked vector | 0.671    |
+| Reranked fts  | 0.843    |
+| Hybrid | 0.849 |
+
+
+
+
--- a/docs/src/hybrid_search/eval.md
+++ b/docs/src/hybrid_search/eval.md
@@ -5,7 +5,9 @@ Hybrid Search is a broad (often misused) term. It can mean anything from combini
 ## The challenge of (re)ranking search results
 Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.
+
 * <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+
 * <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models

 Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
--- a/docs/src/integrations/llamaIndex.md
+++ b/docs/src/integrations/llamaIndex.md
@@ -0,0 +1,139 @@
+# Llama-Index
+![Illustration](../assets/llama-index.jpg)
+
+## Quick start
+You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. You can run the below script to try it out :
+```python
+import logging
+import sys
+
+# Uncomment to see debug logs
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
+
+from llama_index.core import SimpleDirectoryReader, Document, StorageContext
+from llama_index.core import VectorStoreIndex
+from llama_index.vector_stores.lancedb import LanceDBVectorStore
+import textwrap
+import openai
+
+openai.api_key = "sk-..."
+
+documents = SimpleDirectoryReader("./data/your-data-dir/").load_data()
+print("Document ID:", documents[0].doc_id, "Document Hash:", documents[0].hash)
+
+## For LanceDB cloud :
+# vector_store = LanceDBVectorStore( 
+#     uri="db://db_name", # your remote DB URI
+#     api_key="sk_..", # lancedb cloud api key
+#     region="your-region" # the region you configured
+#     ...
+# )
+
+vector_store = LanceDBVectorStore(
+    uri="./lancedb", mode="overwrite", query_type="vector"
+)
+storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+index = VectorStoreIndex.from_documents(
+    documents, storage_context=storage_context
+)
+lance_filter = "metadata.file_name = 'paul_graham_essay.txt' "
+retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
+response = retriever.retrieve("What did the author do growing up?")
+```
+
+### Filtering
+For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
+```python
+from llama_index.core.vector_stores import (
+    MetadataFilters,
+    FilterOperator,
+    FilterCondition,
+    MetadataFilter,
+)
+
+query_filters = MetadataFilters(
+    filters=[
+        MetadataFilter(
+            key="creation_date", operator=FilterOperator.EQ, value="2024-05-23"
+        ),
+        MetadataFilter(
+            key="file_size", value=75040, operator=FilterOperator.GT
+        ),
+    ],
+    condition=FilterCondition.AND,
+)
+```
+
+### Hybrid Search
+For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose.
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+vector_store._add_reranker(reranker)
+
+query_engine = index.as_query_engine(
+    filters=query_filters,
+    vector_store_kwargs={
+        "query_type": "hybrid",
+    }
+)
+
+response = query_engine.query("How much did Viaweb charge per month?")
+```
+
+In the above snippet, you can change/specify query_type again when creating the engine/retriever.
+
+## API reference
+The exhaustive list of parameters for `LanceDBVectorStore` vector store are :  
+- `connection`: Optional, `lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.
+- `uri`: Optional[str], the uri of your database. Defaults to `"/tmp/lancedb"`.
+- `table_name` : Optional[str], Name of your table in the database. Defaults to `"vectors"`.
+- `table`: Optional[Any], `lancedb.db.LanceTable` object to be passed. Defaults to `None`. 
+- `vector_column_name`: Optional[Any], Column name to use for vector's in the table. Defaults to `'vector'`.   
+- `doc_id_key`: Optional[str], Column name to use for document id's in the table. Defaults to `'doc_id'`.  
+- `text_key`: Optional[str], Column name to use for text in the table. Defaults to `'text'`.  
+- `api_key`: Optional[str], API key to use for LanceDB cloud database. Defaults to `None`.  
+- `region`: Optional[str], Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+- `nprobes` : Optional[int], Set the number of probes to use. Only applicable if ANN index is created on the table else its ignored. Defaults to `20`.
+- `refine_factor` : Optional[int], Refine the results by reading extra elements and re-ranking them in memory. Defaults to `None`.
+- `reranker`: Optional[Any], The reranker to use for LanceDB.
+        Defaults to `None`.
+- `overfetch_factor`: Optional[int], The factor by which to fetch more results.
+        Defaults to `1`.
+- `mode`: Optional[str], The mode to use for LanceDB.
+            Defaults to `"overwrite"`.
+- `query_type`:Optional[str], The type of query to use for LanceDB.
+            Defaults to `"vector"`.
+
+
+### Methods 
+
+- __from_table(cls, table: lancedb.db.LanceTable) -> `LanceDBVectorStore`__ : (class method) Creates instance from lancedb table. 
+
+- **_add_reranker(self, reranker: lancedb.rerankers.Reranker) -> `None`** : Add a reranker to an existing vector store. 
+    - Usage :
+        ```python
+        from lancedb.rerankers import ColbertReranker
+        reranker = ColbertReranker()
+        vector_store._add_reranker(reranker)
+        ```               
+- **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
+- __create_index(  
+  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
+) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
+        Make sure your vector column has enough data before creating an index on it.
+
+- __add(self, nodes: `List[BaseNode]`, **add_kwargs: `Any`, ) -> `List[str]`__ :
+adds Nodes to the table
+
+- **delete(self, ref_doc_id: `str`) -> `None`**: Delete nodes using with node_ids.
+- **delete_nodes(self, node_ids: `List[str]`) -> `None`** : Delete nodes using with node_ids.
+- __query(
+        self,
+        query: `VectorStoreQuery`,
+        **kwargs: `Any`,
+    ) -> `VectorStoreQueryResult`__:
+        Query index(`VectorStoreIndex`) for top k most similar nodes. Accepts llamaIndex `VectorStoreQuery` object.
--- a/docs/test/md_testing.py
+++ b/docs/test/md_testing.py
@@ -7,8 +7,7 @@ excluded_globs = [
    "../src/fts.md",
    "../src/embedding.md",
    "../src/examples/*.md",
-    "../src/integrations/voxel51.md",
-    "../src/integrations/langchain.md",
+    "../src/integrations/*.md",
    "../src/guides/tables.md",
    "../src/python/duckdb.md",
    "../src/embeddings/*.md",
@@ -17,6 +16,7 @@ excluded_globs = [
    "../src/basic.md",
    "../src/hybrid_search/hybrid_search.md",
    "../src/reranking/*.md",
+    "../src/guides/tuning_retrievers/*.md",
 ]

 python_prefix = "py"
--- a/java/core/lancedb-jni/src/ffi.rs
+++ b/java/core/lancedb-jni/src/ffi.rs
@@ -175,8 +175,8 @@ impl JNIEnvExt for JNIEnv<'_> {
        if obj.is_null() {
            return Ok(None);
        }
-        let is_empty = self.call_method(obj, "isEmpty", "()Z", &[])?;
-        if is_empty.z()? {
+        let is_present = self.call_method(obj, "isPresent", "()Z", &[])?;
+        if !is_present.z()? {
            // TODO(lu): put get java object into here cuz can only get java Object
            Ok(None)
        } else {
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.5.1",
+  "version": "0.5.2",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.5.1",
+      "version": "0.5.2",
      "cpu": [
        "x64",
        "arm64"
--- a/node/package.json
+++ b/node/package.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.5.1",
+  "version": "0.6.0",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
  "scripts": {
    "tsc": "tsc -b",
-    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb-node index.node -- cargo build --message-format=json",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb_node index.node -- cargo build --message-format=json",
    "build-release": "npm run build -- --release",
    "test": "npm run tsc && mocha -recursive dist/test",
    "integration-test": "npm run tsc && mocha -recursive dist/integration_test",
--- a/node/src/index.ts
+++ b/node/src/index.ts
@@ -695,18 +695,26 @@ export interface MergeInsertArgs {
  whenNotMatchedBySourceDelete?: string | boolean
 }

+export enum IndexStatus {
+  Pending = "pending",
+  Indexing = "indexing",
+  Done = "done",
+  Failed = "failed"
+}
+
 export interface VectorIndex {
  columns: string[]
  name: string
  uuid: string
+  status: IndexStatus
 }

 export interface IndexStats {
  numIndexedRows: number | null
  numUnindexedRows: number | null
-  index_type: string | null
-  distance_type: string | null
-  completed_at: string | null
+  indexType: string | null
+  distanceType: string | null
+  completedAt: string | null
 }

 /**
--- a/node/src/remote/index.ts
+++ b/node/src/remote/index.ts
@@ -522,9 +522,9 @@ export class RemoteTable<T = number[]> implements Table<T> {
    return {
      numIndexedRows: body?.num_indexed_rows,
      numUnindexedRows: body?.num_unindexed_rows,
-      index_type: body?.index_type,
-      distance_type: body?.distance_type,
-      completed_at: body?.completed_at
+      indexType: body?.index_type,
+      distanceType: body?.distance_type,
+      completedAt: body?.completed_at
    }
  }

--- a/nodejs/test/connection.test.ts
+++ b/nodejs/test/connection.test.ts
@@ -57,6 +57,18 @@ describe("given a connection", () => {
    expect(db.isOpen()).toBe(false);
    await expect(db.tableNames()).rejects.toThrow("Connection is closed");
  });
+  it("should be able to create a table from an object arg `createTable(options)`, or args `createTable(name, data, options)`", async () => {
+    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+    await expect(tbl.countRows()).resolves.toBe(2);
+
+    tbl = await db.createTable({
+      name: "test",
+      data: [{ id: 3 }],
+      mode: "overwrite",
+    });
+
+    await expect(tbl.countRows()).resolves.toBe(1);
+  });

  it("should fail if creating table twice, unless overwrite is true", async () => {
    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
--- a/nodejs/test/embedding.test.ts
+++ b/nodejs/test/embedding.test.ts
@@ -230,7 +230,7 @@ describe("embedding functions", () => {
    },
  );

-  test.only.each([new Float16(), new Float32(), new Float64()])(
+  test.each([new Float16(), new Float32(), new Float64()])(
    "should be able to provide auto embeddings with multiple float datatypes",
    async (floatType) => {
      @register("test1")
--- a/nodejs/test/table.test.ts
+++ b/nodejs/test/table.test.ts
@@ -132,6 +132,140 @@ describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
  });
 });

+describe("merge insert", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const conn = await connect(tmpDir.name);
+
+    table = await conn.createTable("some_table", [
+      { a: 1, b: "a" },
+      { a: 2, b: "b" },
+      { a: 3, b: "c" },
+    ]);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  test("upsert", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute(newData);
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+
+    expect(
+      JSON.parse(JSON.stringify((await table.toArrow()).toArray())),
+    ).toEqual(expected);
+  });
+  test("conditional update", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll({ where: "target.b = 'b'" })
+      .execute(newData);
+
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 3, b: "c" },
+    ];
+    // round trip to arrow and back to json to avoid comparing arrow objects to js object
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+
+    expect(res).toEqual(expected);
+  });
+
+  test("insert if not exists", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table.mergeInsert("a").whenNotMatchedInsertAll().execute(newData);
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "b" },
+      { a: 3, b: "c" },
+      { a: 4, b: "z" },
+    ];
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+  test("replace range", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .whenNotMatchedBySourceDelete({ where: "a > 2" })
+      .execute(newData);
+
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+  test("replace range no condition", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .whenNotMatchedBySourceDelete()
+      .execute(newData);
+
+    const expected = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+});
+
 describe("When creating an index", () => {
  let tmpDir: tmp.DirResult;
  const schema = new Schema([
@@ -171,6 +305,7 @@ describe("When creating an index", () => {
    const indices = await tbl.listIndices();
    expect(indices.length).toBe(1);
    expect(indices[0]).toEqual({
+      name: "vec_idx",
      indexType: "IvfPq",
      columns: ["vec"],
    });
@@ -227,6 +362,24 @@ describe("When creating an index", () => {
    for await (const r of tbl.query().where("id > 1").select(["id"])) {
      expect(r.numRows).toBe(298);
    }
+    // should also work with 'filter' alias
+    for await (const r of tbl.query().filter("id > 1").select(["id"])) {
+      expect(r.numRows).toBe(298);
+    }
+  });
+
+  test("should be able to get index stats", async () => {
+    await tbl.createIndex("id");
+
+    const stats = await tbl.indexStats("id_idx");
+    expect(stats).toBeDefined();
+    expect(stats?.numIndexedRows).toEqual(300);
+    expect(stats?.numUnindexedRows).toEqual(0);
+  });
+
+  test("when getting stats on non-existent index", async () => {
+    const stats = await tbl.indexStats("some non-existent index");
+    expect(stats).toBeUndefined();
  });

  // TODO: Move this test to the query API test (making sure we can reject queries
--- a/nodejs/biome.json
+++ b/nodejs/biome.json
@@ -77,7 +77,7 @@
        "noDuplicateObjectKeys": "error",
        "noDuplicateParameters": "error",
        "noEmptyBlockStatements": "error",
-        "noExplicitAny": "error",
+        "noExplicitAny": "warn",
        "noExtraNonNullAssertion": "error",
        "noFallthroughSwitchClause": "error",
        "noFunctionAssign": "error",
--- a/nodejs/lancedb/connection.ts
+++ b/nodejs/lancedb/connection.ts
@@ -12,38 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-import { Table as ArrowTable, Schema } from "./arrow";
-import {
-  fromTableToBuffer,
-  isArrowTable,
-  makeArrowTable,
-  makeEmptyTable,
-} from "./arrow";
+import { Table as ArrowTable, Data, Schema } from "./arrow";
+import { fromTableToBuffer, makeEmptyTable } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
-import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
-import { Table } from "./table";
-
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accepted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param {string} uri - The uri of the database. If the database uri starts
- * with `db://` then it connects to a remote database.
- * @see {@link ConnectionOptions} for more details on the URI format.
- */
-export async function connect(
-  uri: string,
-  opts?: Partial<ConnectionOptions>,
-): Promise<Connection> {
-  opts = opts ?? {};
-  opts.storageOptions = cleanseStorageOptions(opts.storageOptions);
-  const nativeConn = await LanceDbConnection.new(uri, opts);
-  return new Connection(nativeConn);
-}
+import { Connection as LanceDbConnection } from "./native";
+import { LocalTable, Table } from "./table";

 export interface CreateTableOptions {
  /**
@@ -117,7 +90,6 @@ export interface TableNamesOptions {
  /** An optional limit to the number of results to return. */
  limit?: number;
 }
-
 /**
 * A LanceDB Connection that allows you to open tables and create new ones.
 *
@@ -136,17 +108,15 @@ export interface TableNamesOptions {
 * Any created tables are independent and will continue to work even if
 * the underlying connection has been closed.
 */
-export class Connection {
-  readonly inner: LanceDbConnection;
-
-  constructor(inner: LanceDbConnection) {
-    this.inner = inner;
+export abstract class Connection {
+  [Symbol.for("nodejs.util.inspect.custom")](): string {
+    return this.display();
  }

-  /** Return true if the connection has not been closed */
-  isOpen(): boolean {
-    return this.inner.isOpen();
-  }
+  /**
+   * Return true if the connection has not been closed
+   */
+  abstract isOpen(): boolean;

  /**
   * Close the connection, releasing any underlying resources.
@@ -155,14 +125,12 @@ export class Connection {
   *
   * Any attempt to use the connection after it is closed will result in an error.
   */
-  close(): void {
-    this.inner.close();
-  }
+  abstract close(): void;

-  /** Return a brief description of the connection */
-  display(): string {
-    return this.inner.display();
-  }
+  /**
+   * Return a brief description of the connection
+   */
+  abstract display(): string;

  /**
   * List all the table names in this database.
@@ -170,15 +138,86 @@ export class Connection {
   * Tables will be returned in lexicographical order.
   * @param {Partial<TableNamesOptions>} options - options to control the
   * paging / start point
+   *
   */
-  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
-    return this.inner.tableNames(options?.startAfter, options?.limit);
-  }
+  abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;

  /**
   * Open a table in the database.
   * @param {string} name - The name of the table
   */
+  abstract openTable(
+    name: string,
+    options?: Partial<OpenTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {object} options - The options object.
+   * @param {string} options.name - The name of the table.
+   * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
+   *
+   */
+  abstract createTable(
+    options: {
+      name: string;
+      data: Data;
+    } & Partial<CreateTableOptions>,
+  ): Promise<Table>;
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {string} name - The name of the table.
+   * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
+   * to be inserted into the table
+   */
+  abstract createTable(
+    name: string,
+    data: Record<string, unknown>[] | ArrowTable,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Creates a new empty Table
+   * @param {string} name - The name of the table.
+   * @param {Schema} schema - The schema of the table
+   */
+  abstract createEmptyTable(
+    name: string,
+    schema: Schema,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Drop an existing table.
+   * @param {string} name The name of the table to drop.
+   */
+  abstract dropTable(name: string): Promise<void>;
+}
+
+export class LocalConnection extends Connection {
+  readonly inner: LanceDbConnection;
+
+  constructor(inner: LanceDbConnection) {
+    super();
+    this.inner = inner;
+  }
+
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  close(): void {
+    this.inner.close();
+  }
+
+  display(): string {
+    return this.inner.display();
+  }
+
+  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+    return this.inner.tableNames(options?.startAfter, options?.limit);
+  }
+
  async openTable(
    name: string,
    options?: Partial<OpenTableOptions>,
@@ -189,55 +228,35 @@ export class Connection {
      options?.indexCacheSize,
    );

-    return new Table(innerTable);
+    return new LocalTable(innerTable);
  }

-  /**
-   * Creates a new Table and initialize it with new data.
-   * @param {string} name - The name of the table.
-   * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
-   * to be inserted into the table
-   */
  async createTable(
-    name: string,
-    data: Record<string, unknown>[] | ArrowTable,
+    nameOrOptions:
+      | string
+      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
+    data?: Record<string, unknown>[] | ArrowTable,
    options?: Partial<CreateTableOptions>,
  ): Promise<Table> {
-    let mode: string = options?.mode ?? "create";
-    const existOk = options?.existOk ?? false;
-
-    if (mode === "create" && existOk) {
-      mode = "exist_ok";
+    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+      const { name, data, ...options } = nameOrOptions;
+      return this.createTable(name, data, options);
    }
-
-    let table: ArrowTable;
-    if (isArrowTable(data)) {
-      table = data;
-    } else {
-      table = makeArrowTable(data, options);
+    if (data === undefined) {
+      throw new Error("data is required");
    }
-
-    const buf = await fromTableToBuffer(
-      table,
-      options?.embeddingFunction,
-      options?.schema,
-    );
+    const { buf, mode } = await Table.parseTableData(data, options);
    const innerTable = await this.inner.createTable(
-      name,
+      nameOrOptions,
      buf,
      mode,
      cleanseStorageOptions(options?.storageOptions),
      options?.useLegacyFormat,
    );

-    return new Table(innerTable);
+    return new LocalTable(innerTable);
  }

-  /**
-   * Creates a new empty Table
-   * @param {string} name - The name of the table.
-   * @param {Schema} schema - The schema of the table
-   */
  async createEmptyTable(
    name: string,
    schema: Schema,
@@ -265,13 +284,9 @@ export class Connection {
      cleanseStorageOptions(options?.storageOptions),
      options?.useLegacyFormat,
    );
-    return new Table(innerTable);
+    return new LocalTable(innerTable);
  }

-  /**
-   * Drop an existing table.
-   * @param {string} name The name of the table to drop.
-   */
  async dropTable(name: string): Promise<void> {
    return this.inner.dropTable(name);
  }
@@ -280,7 +295,7 @@ export class Connection {
 /**
 * Takes storage options and makes all the keys snake case.
 */
-function cleanseStorageOptions(
+export function cleanseStorageOptions(
  options?: Record<string, string>,
 ): Record<string, string> | undefined {
  if (options === undefined) {
--- a/nodejs/lancedb/index.ts
+++ b/nodejs/lancedb/index.ts
@@ -12,25 +12,43 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+import {
+  Connection,
+  LocalConnection,
+  cleanseStorageOptions,
+} from "./connection";
+
+import {
+  ConnectionOptions,
+  Connection as LanceDbConnection,
+} from "./native.js";
+
+import { RemoteConnection, RemoteConnectionOptions } from "./remote";
+
 export {
  WriteOptions,
  WriteMode,
  AddColumnsSql,
  ColumnAlteration,
  ConnectionOptions,
+  IndexStatistics,
+  IndexMetadata,
+  IndexConfig,
 } from "./native.js";
+
 export {
  makeArrowTable,
  MakeArrowTableOptions,
  Data,
  VectorColumnOptions,
 } from "./arrow";
+
 export {
-  connect,
  Connection,
  CreateTableOptions,
  TableNamesOptions,
 } from "./connection";
+
 export {
  ExecutableQuery,
  Query,
@@ -38,6 +56,87 @@ export {
  VectorQuery,
  RecordBatchIterator,
 } from "./query";
+
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
-export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+
+export { Table, AddDataOptions, UpdateOptions } from "./table";
+
 export * as embedding from "./embedding";
+
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect("/path/to/database");
+ * ```
+ * @example
+ * ```ts
+ * const conn = await connect(
+ *   "s3://bucket/path/to/database",
+ *   {storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export async function connect(
+  uri: string,
+  opts?: Partial<ConnectionOptions | RemoteConnectionOptions>,
+): Promise<Connection>;
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param  options - The options to use when connecting to the database
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export async function connect(
+  opts: Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string },
+): Promise<Connection>;
+export async function connect(
+  uriOrOptions:
+    | string
+    | (Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string }),
+  opts: Partial<ConnectionOptions | RemoteConnectionOptions> = {},
+): Promise<Connection> {
+  let uri: string | undefined;
+  if (typeof uriOrOptions !== "string") {
+    const { uri: uri_, ...options } = uriOrOptions;
+    uri = uri_;
+    opts = options;
+  } else {
+    uri = uriOrOptions;
+  }
+
+  if (!uri) {
+    throw new Error("uri is required");
+  }
+
+  if (uri?.startsWith("db://")) {
+    return new RemoteConnection(uri, opts as RemoteConnectionOptions);
+  }
+  opts = (opts as ConnectionOptions) ?? {};
+  (<ConnectionOptions>opts).storageOptions = cleanseStorageOptions(
+    (<ConnectionOptions>opts).storageOptions,
+  );
+  const nativeConn = await LanceDbConnection.new(uri, opts);
+  return new LocalConnection(nativeConn);
+}
--- a/nodejs/lancedb/merge.ts
+++ b/nodejs/lancedb/merge.ts
@@ -0,0 +1,70 @@
+import { Data, fromDataToBuffer } from "./arrow";
+import { NativeMergeInsertBuilder } from "./native";
+
+/** A builder used to create and run a merge insert operation */
+export class MergeInsertBuilder {
+  #native: NativeMergeInsertBuilder;
+
+  /** Construct a MergeInsertBuilder. __Internal use only.__ */
+  constructor(native: NativeMergeInsertBuilder) {
+    this.#native = native;
+  }
+
+  /**
+   * Rows that exist in both the source table (new data) and
+   * the target table (old data) will be updated, replacing
+   * the old row with the corresponding matching row.
+   *
+   * If there are multiple matches then the behavior is undefined.
+   * Currently this causes multiple copies of the row to be created
+   * but that behavior is subject to change.
+   *
+   * An optional condition may be specified.  If it is, then only
+   * matched rows that satisfy the condtion will be updated.  Any
+   * rows that do not satisfy the condition will be left as they
+   * are.  Failing to satisfy the condition does not cause a
+   * "matched row" to become a "not matched" row.
+   *
+   * The condition should be an SQL string.  Use the prefix
+   * target. to refer to rows in the target table (old data)
+   * and the prefix source. to refer to rows in the source
+   * table (new data).
+   *
+   * For example, "target.last_update < source.last_update"
+   */
+  whenMatchedUpdateAll(options?: { where: string }): MergeInsertBuilder {
+    return new MergeInsertBuilder(
+      this.#native.whenMatchedUpdateAll(options?.where),
+    );
+  }
+  /**
+   * Rows that exist only in the source table (new data) should
+   * be inserted into the target table.
+   */
+  whenNotMatchedInsertAll(): MergeInsertBuilder {
+    return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
+  }
+  /**
+   * Rows that exist only in the target table (old data) will be
+   * deleted.  An optional condition can be provided to limit what
+   * data is deleted.
+   *
+   * @param options.where - An optional condition to limit what data is deleted
+   */
+  whenNotMatchedBySourceDelete(options?: {
+    where: string;
+  }): MergeInsertBuilder {
+    return new MergeInsertBuilder(
+      this.#native.whenNotMatchedBySourceDelete(options?.where),
+    );
+  }
+  /**
+   * Executes the merge insert operation
+   *
+   * Nothing is returned but the `Table` is updated
+   */
+  async execute(data: Data): Promise<void> {
+    const buffer = await fromDataToBuffer(data);
+    await this.#native.execute(buffer);
+  }
+}
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -114,6 +114,14 @@ export class QueryBase<
    this.inner.onlyIf(predicate);
    return this as unknown as QueryType;
  }
+  /**
+   * A filter statement to be applied to this query.
+   * @alias where
+   * @deprecated Use `where` instead
+   */
+  filter(predicate: string): QueryType {
+    return this.where(predicate);
+  }

  /**
   * Return only the specified columns.
--- a/nodejs/lancedb/remote/client.ts
+++ b/nodejs/lancedb/remote/client.ts
@@ -0,0 +1,221 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios, {
+  AxiosError,
+  type AxiosResponse,
+  type ResponseType,
+} from "axios";
+import { Table as ArrowTable } from "../arrow";
+import { tableFromIPC } from "../arrow";
+import { VectorQuery } from "../query";
+
+export class RestfulLanceDBClient {
+  #dbName: string;
+  #region: string;
+  #apiKey: string;
+  #hostOverride?: string;
+  #closed: boolean = false;
+  #connectionTimeout: number = 12 * 1000; // 12 seconds;
+  #readTimeout: number = 30 * 1000; // 30 seconds;
+  #session?: import("axios").AxiosInstance;
+
+  constructor(
+    dbName: string,
+    apiKey: string,
+    region: string,
+    hostOverride?: string,
+    connectionTimeout?: number,
+    readTimeout?: number,
+  ) {
+    this.#dbName = dbName;
+    this.#apiKey = apiKey;
+    this.#region = region;
+    this.#hostOverride = hostOverride ?? this.#hostOverride;
+    this.#connectionTimeout = connectionTimeout ?? this.#connectionTimeout;
+    this.#readTimeout = readTimeout ?? this.#readTimeout;
+  }
+
+  // todo: cache the session.
+  get session(): import("axios").AxiosInstance {
+    if (this.#session !== undefined) {
+      return this.#session;
+    } else {
+      return axios.create({
+        baseURL: this.url,
+        headers: {
+          // biome-ignore lint/style/useNamingConvention: external api
+          Authorization: `Bearer ${this.#apiKey}`,
+        },
+        transformResponse: decodeErrorData,
+        timeout: this.#connectionTimeout,
+      });
+    }
+  }
+
+  get url(): string {
+    return (
+      this.#hostOverride ??
+      `https://${this.#dbName}.${this.#region}.api.lancedb.com`
+    );
+  }
+
+  get headers(): { [key: string]: string } {
+    const headers: { [key: string]: string } = {
+      "x-api-key": this.#apiKey,
+      "x-request-id": "na",
+    };
+    if (this.#region == "local") {
+      headers["Host"] = `${this.#dbName}.${this.#region}.api.lancedb.com`;
+    }
+    if (this.#hostOverride) {
+      headers["x-lancedb-database"] = this.#dbName;
+    }
+    return headers;
+  }
+
+  isOpen(): boolean {
+    return !this.#closed;
+  }
+
+  private checkNotClosed(): void {
+    if (this.#closed) {
+      throw new Error("Connection is closed");
+    }
+  }
+
+  close(): void {
+    this.#session = undefined;
+    this.#closed = true;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  async get(uri: string, params?: Record<string, any>): Promise<any> {
+    this.checkNotClosed();
+    uri = new URL(uri, this.url).toString();
+    let response;
+    try {
+      response = await this.session.get(uri, {
+        headers: this.headers,
+        params,
+      });
+    } catch (e) {
+      if (e instanceof AxiosError) {
+        response = e.response;
+      } else {
+        throw e;
+      }
+    }
+
+    RestfulLanceDBClient.checkStatus(response!);
+    return response!.data;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: api response
+  async post(uri: string, body?: any): Promise<any>;
+  async post(
+    uri: string,
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body: any,
+    additional: {
+      config?: { responseType: "arraybuffer" };
+      headers?: Record<string, string>;
+      params?: Record<string, string>;
+    },
+  ): Promise<Buffer>;
+  async post(
+    uri: string,
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body?: any,
+    additional?: {
+      config?: { responseType: ResponseType };
+      headers?: Record<string, string>;
+      params?: Record<string, string>;
+    },
+    // biome-ignore lint/suspicious/noExplicitAny: api response
+  ): Promise<any> {
+    this.checkNotClosed();
+    uri = new URL(uri, this.url).toString();
+    additional = Object.assign(
+      { config: { responseType: "json" } },
+      additional,
+    );
+
+    const headers = { ...this.headers, ...additional.headers };
+
+    if (!headers["Content-Type"]) {
+      headers["Content-Type"] = "application/json";
+    }
+    let response;
+    try {
+      response = await this.session.post(uri, body, {
+        headers,
+        responseType: additional!.config!.responseType,
+        params: new Map(Object.entries(additional.params ?? {})),
+      });
+    } catch (e) {
+      if (e instanceof AxiosError) {
+        response = e.response;
+      } else {
+        throw e;
+      }
+    }
+    RestfulLanceDBClient.checkStatus(response!);
+    if (additional!.config!.responseType === "arraybuffer") {
+      return response!.data;
+    } else {
+      return JSON.parse(response!.data);
+    }
+  }
+
+  async listTables(limit = 10, pageToken = ""): Promise<string[]> {
+    const json = await this.get("/v1/table", { limit, pageToken });
+    return json.tables;
+  }
+
+  async query(tableName: string, query: VectorQuery): Promise<ArrowTable> {
+    const tbl = await this.post(`/v1/table/${tableName}/query`, query, {
+      config: {
+        responseType: "arraybuffer",
+      },
+    });
+    return tableFromIPC(tbl);
+  }
+
+  static checkStatus(response: AxiosResponse): void {
+    if (response.status === 404) {
+      throw new Error(`Not found: ${response.data}`);
+    } else if (response.status >= 400 && response.status < 500) {
+      throw new Error(
+        `Bad Request: ${response.status}, error: ${response.data}`,
+      );
+    } else if (response.status >= 500 && response.status < 600) {
+      throw new Error(
+        `Internal Server Error: ${response.status}, error: ${response.data}`,
+      );
+    } else if (response.status !== 200) {
+      throw new Error(
+        `Unknown Error: ${response.status}, error: ${response.data}`,
+      );
+    }
+  }
+}
+
+function decodeErrorData(data: unknown) {
+  if (Buffer.isBuffer(data)) {
+    const decoded = data.toString("utf-8");
+    return decoded;
+  }
+  return data;
+}
--- a/nodejs/lancedb/remote/connection.ts
+++ b/nodejs/lancedb/remote/connection.ts
@@ -0,0 +1,196 @@
+import { Schema } from "apache-arrow";
+import { Data, fromTableToStreamBuffer, makeEmptyTable } from "../arrow";
+import {
+  Connection,
+  CreateTableOptions,
+  OpenTableOptions,
+  TableNamesOptions,
+} from "../connection";
+import { Table } from "../table";
+import { TTLCache } from "../util";
+import { RestfulLanceDBClient } from "./client";
+import { RemoteTable } from "./table";
+
+export interface RemoteConnectionOptions {
+  apiKey?: string;
+  region?: string;
+  hostOverride?: string;
+  connectionTimeout?: number;
+  readTimeout?: number;
+}
+
+export class RemoteConnection extends Connection {
+  #dbName: string;
+  #apiKey: string;
+  #region: string;
+  #client: RestfulLanceDBClient;
+  #tableCache = new TTLCache(300_000);
+
+  constructor(
+    url: string,
+    {
+      apiKey,
+      region,
+      hostOverride,
+      connectionTimeout,
+      readTimeout,
+    }: RemoteConnectionOptions,
+  ) {
+    super();
+    apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
+    region = region ?? process.env.LANCEDB_REGION;
+
+    if (!apiKey) {
+      throw new Error("apiKey is required when connecting to LanceDB Cloud");
+    }
+
+    if (!region) {
+      throw new Error("region is required when connecting to LanceDB Cloud");
+    }
+
+    const parsed = new URL(url);
+    if (parsed.protocol !== "db:") {
+      throw new Error(
+        `invalid protocol: ${parsed.protocol}, only accepts db://`,
+      );
+    }
+
+    this.#dbName = parsed.hostname;
+    this.#apiKey = apiKey;
+    this.#region = region;
+    this.#client = new RestfulLanceDBClient(
+      this.#dbName,
+      this.#apiKey,
+      this.#region,
+      hostOverride,
+      connectionTimeout,
+      readTimeout,
+    );
+  }
+
+  isOpen(): boolean {
+    return this.#client.isOpen();
+  }
+  close(): void {
+    return this.#client.close();
+  }
+
+  display(): string {
+    return `RemoteConnection(${this.#dbName})`;
+  }
+
+  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+    const response = await this.#client.get("/v1/table/", {
+      limit: options?.limit ?? 10,
+      // biome-ignore lint/style/useNamingConvention: <explanation>
+      page_token: options?.startAfter ?? "",
+    });
+    const body = await response.body();
+    for (const table of body.tables) {
+      this.#tableCache.set(table, true);
+    }
+    return body.tables;
+  }
+
+  async openTable(
+    name: string,
+    _options?: Partial<OpenTableOptions> | undefined,
+  ): Promise<Table> {
+    if (this.#tableCache.get(name) === undefined) {
+      await this.#client.post(
+        `/v1/table/${encodeURIComponent(name)}/describe/`,
+      );
+      this.#tableCache.set(name, true);
+    }
+    return new RemoteTable(this.#client, name, this.#dbName);
+  }
+
+  async createTable(
+    nameOrOptions:
+      | string
+      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
+    data?: Data,
+    options?: Partial<CreateTableOptions> | undefined,
+  ): Promise<Table> {
+    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+      const { name, data, ...options } = nameOrOptions;
+      return this.createTable(name, data, options);
+    }
+    if (data === undefined) {
+      throw new Error("data is required");
+    }
+    if (options?.mode) {
+      console.warn(
+        "option 'mode' is not supported in LanceDB Cloud",
+        "LanceDB Cloud only supports the default 'create' mode.",
+        "If the table already exists, an error will be thrown.",
+      );
+    }
+    if (options?.embeddingFunction) {
+      console.warn(
+        "embedding_functions is not yet supported on LanceDB Cloud.",
+        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
+        "for this feature.",
+      );
+    }
+
+    const { buf } = await Table.parseTableData(
+      data,
+      options,
+      true /** streaming */,
+    );
+
+    await this.#client.post(
+      `/v1/table/${encodeURIComponent(nameOrOptions)}/create/`,
+      buf,
+      {
+        config: {
+          responseType: "arraybuffer",
+        },
+        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+      },
+    );
+    this.#tableCache.set(nameOrOptions, true);
+    return new RemoteTable(this.#client, nameOrOptions, this.#dbName);
+  }
+
+  async createEmptyTable(
+    name: string,
+    schema: Schema,
+    options?: Partial<CreateTableOptions> | undefined,
+  ): Promise<Table> {
+    if (options?.mode) {
+      console.warn(`mode is not supported on LanceDB Cloud`);
+    }
+
+    if (options?.embeddingFunction) {
+      console.warn(
+        "embeddingFunction is not yet supported on LanceDB Cloud.",
+        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
+        "for this feature.",
+      );
+    }
+    const emptyTable = makeEmptyTable(schema);
+    const buf = await fromTableToStreamBuffer(emptyTable);
+
+    await this.#client.post(
+      `/v1/table/${encodeURIComponent(name)}/create/`,
+      buf,
+      {
+        config: {
+          responseType: "arraybuffer",
+        },
+        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+      },
+    );
+
+    this.#tableCache.set(name, true);
+    return new RemoteTable(this.#client, name, this.#dbName);
+  }
+
+  async dropTable(name: string): Promise<void> {
+    await this.#client.post(`/v1/table/${encodeURIComponent(name)}/drop/`);
+
+    this.#tableCache.delete(name);
+  }
+}
--- a/nodejs/lancedb/remote/index.ts
+++ b/nodejs/lancedb/remote/index.ts
@@ -0,0 +1,3 @@
+export { RestfulLanceDBClient } from "./client";
+export { type RemoteConnectionOptions, RemoteConnection } from "./connection";
+export { RemoteTable } from "./table";
--- a/nodejs/lancedb/remote/table.ts
+++ b/nodejs/lancedb/remote/table.ts
@@ -0,0 +1,172 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Table as ArrowTable } from "apache-arrow";
+
+import { Data, IntoVector } from "../arrow";
+
+import { IndexStatistics } from "..";
+import { CreateTableOptions } from "../connection";
+import { IndexOptions } from "../indices";
+import { MergeInsertBuilder } from "../merge";
+import { VectorQuery } from "../query";
+import { AddDataOptions, Table, UpdateOptions } from "../table";
+import { RestfulLanceDBClient } from "./client";
+
+export class RemoteTable extends Table {
+  #client: RestfulLanceDBClient;
+  #name: string;
+
+  // Used in the display() method
+  #dbName: string;
+
+  get #tablePrefix() {
+    return `/v1/table/${encodeURIComponent(this.#name)}/`;
+  }
+
+  get name(): string {
+    return this.#name;
+  }
+
+  public constructor(
+    client: RestfulLanceDBClient,
+    tableName: string,
+    dbName: string,
+  ) {
+    super();
+    this.#client = client;
+    this.#name = tableName;
+    this.#dbName = dbName;
+  }
+
+  isOpen(): boolean {
+    return !this.#client.isOpen();
+  }
+
+  close(): void {
+    this.#client.close();
+  }
+
+  display(): string {
+    return `RemoteTable(${this.#dbName}; ${this.#name})`;
+  }
+
+  async schema(): Promise<import("apache-arrow").Schema> {
+    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+    // TODO: parse this into a valid arrow schema
+    return resp.schema;
+  }
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+    const { buf, mode } = await Table.parseTableData(
+      data,
+      options as CreateTableOptions,
+      true,
+    );
+    await this.#client.post(`${this.#tablePrefix}/insert/`, buf, {
+      params: {
+        mode,
+      },
+      headers: {
+        "Content-Type": "application/vnd.apache.arrow.stream",
+      },
+    });
+  }
+
+  async update(
+    updates: Map<string, string> | Record<string, string>,
+    options?: Partial<UpdateOptions>,
+  ): Promise<void> {
+    await this.#client.post(`${this.#tablePrefix}/update/`, {
+      predicate: options?.where ?? null,
+      updates: Object.entries(updates).map(([key, value]) => [key, value]),
+    });
+  }
+  async countRows(filter?: unknown): Promise<number> {
+    const payload = { predicate: filter };
+    return await this.#client.post(`${this.#tablePrefix}/count_rows/`, payload);
+  }
+
+  async delete(predicate: unknown): Promise<void> {
+    const payload = { predicate };
+    await this.#client.post(`${this.#tablePrefix}/delete/`, payload);
+  }
+  async createIndex(
+    column: string,
+    options?: Partial<IndexOptions>,
+  ): Promise<void> {
+    if (options !== undefined) {
+      console.warn("options are not yet supported on the LanceDB cloud");
+    }
+    const indexType = "vector";
+    const metric = "L2";
+    const data = {
+      column,
+      // biome-ignore lint/style/useNamingConvention: external API
+      index_type: indexType,
+      // biome-ignore lint/style/useNamingConvention: external API
+      metric_type: metric,
+    };
+    await this.#client.post(`${this.#tablePrefix}/create_index`, data);
+  }
+  query(): import("..").Query {
+    throw new Error("query() is not yet supported on the LanceDB cloud");
+  }
+  search(query: IntoVector): VectorQuery;
+  search(query: string): Promise<VectorQuery>;
+  search(_query: string | IntoVector): VectorQuery | Promise<VectorQuery> {
+    throw new Error("search() is not yet supported on the LanceDB cloud");
+  }
+  vectorSearch(_vector: unknown): import("..").VectorQuery {
+    throw new Error("vectorSearch() is not yet supported on the LanceDB cloud");
+  }
+  addColumns(_newColumnTransforms: unknown): Promise<void> {
+    throw new Error("addColumns() is not yet supported on the LanceDB cloud");
+  }
+  alterColumns(_columnAlterations: unknown): Promise<void> {
+    throw new Error("alterColumns() is not yet supported on the LanceDB cloud");
+  }
+  dropColumns(_columnNames: unknown): Promise<void> {
+    throw new Error("dropColumns() is not yet supported on the LanceDB cloud");
+  }
+  async version(): Promise<number> {
+    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+    return resp.version;
+  }
+  checkout(_version: unknown): Promise<void> {
+    throw new Error("checkout() is not yet supported on the LanceDB cloud");
+  }
+  checkoutLatest(): Promise<void> {
+    throw new Error(
+      "checkoutLatest() is not yet supported on the LanceDB cloud",
+    );
+  }
+  restore(): Promise<void> {
+    throw new Error("restore() is not yet supported on the LanceDB cloud");
+  }
+  optimize(_options?: unknown): Promise<import("../native").OptimizeStats> {
+    throw new Error("optimize() is not yet supported on the LanceDB cloud");
+  }
+  async listIndices(): Promise<import("../native").IndexConfig[]> {
+    return await this.#client.post(`${this.#tablePrefix}/index/list/`);
+  }
+  toArrow(): Promise<ArrowTable> {
+    throw new Error("toArrow() is not yet supported on the LanceDB cloud");
+  }
+  mergeInsert(_on: string | string[]): MergeInsertBuilder {
+    throw new Error("mergeInsert() is not yet supported on the LanceDB cloud");
+  }
+  async indexStats(_name: string): Promise<IndexStatistics | undefined> {
+    throw new Error("indexStats() is not yet supported on the LanceDB cloud");
+  }
+}
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -18,20 +18,26 @@ import {
  IntoVector,
  Schema,
  fromDataToBuffer,
+  fromTableToBuffer,
+  fromTableToStreamBuffer,
+  isArrowTable,
+  makeArrowTable,
  tableFromIPC,
 } from "./arrow";
+import { CreateTableOptions } from "./connection";

 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
+import { MergeInsertBuilder } from "./merge";
 import {
  AddColumnsSql,
  ColumnAlteration,
  IndexConfig,
+  IndexStatistics,
  OptimizeStats,
  Table as _NativeTable,
 } from "./native";
 import { Query, VectorQuery } from "./query";
-export { IndexConfig } from "./native";

 /**
 * Options for adding data to a table.
@@ -88,19 +94,15 @@ export interface OptimizeOptions {
 * Closing a table is optional.  It not closed, it will be closed when it is garbage
 * collected.
 */
-export class Table {
-  private readonly inner: _NativeTable;
-
-  /** Construct a Table. Internal use only. */
-  constructor(inner: _NativeTable) {
-    this.inner = inner;
+export abstract class Table {
+  [Symbol.for("nodejs.util.inspect.custom")](): string {
+    return this.display();
  }
+  /** Returns the name of the table */
+  abstract get name(): string;

  /** Return true if the table has not been closed */
-  isOpen(): boolean {
-    return this.inner.isOpen();
-  }
-
+  abstract isOpen(): boolean;
  /**
   * Close the table, releasing any underlying resources.
   *
@@ -108,48 +110,16 @@ export class Table {
   *
   * Any attempt to use the table after it is closed will result in an error.
   */
-  close(): void {
-    this.inner.close();
-  }
-
+  abstract close(): void;
  /** Return a brief description of the table */
-  display(): string {
-    return this.inner.display();
-  }
-
-  async #getEmbeddingFunctions(): Promise<
-    Map<string, EmbeddingFunctionConfig>
-  > {
-    const schema = await this.schema();
-    const registry = getRegistry();
-    return registry.parseFunctions(schema.metadata);
-  }
-
+  abstract display(): string;
  /** Get the schema of the table. */
-  async schema(): Promise<Schema> {
-    const schemaBuf = await this.inner.schema();
-    const tbl = tableFromIPC(schemaBuf);
-    return tbl.schema;
-  }
-
+  abstract schema(): Promise<Schema>;
  /**
   * Insert records into this Table.
   * @param {Data} data Records to be inserted into the Table
   */
-  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
-    const mode = options?.mode ?? "append";
-    const schema = await this.schema();
-    const registry = getRegistry();
-    const functions = registry.parseFunctions(schema.metadata);
-
-    const buffer = await fromDataToBuffer(
-      data,
-      functions.values().next().value,
-      schema,
-    );
-    await this.inner.add(buffer, mode);
-  }
-
+  abstract add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
  /**
   * Update existing records in the Table
   *
@@ -175,30 +145,14 @@ export class Table {
   * @param {Partial<UpdateOptions>} options - additional options to control
   * the update behavior
   */
-  async update(
+  abstract update(
    updates: Map<string, string> | Record<string, string>,
    options?: Partial<UpdateOptions>,
-  ) {
-    const onlyIf = options?.where;
-    let columns: [string, string][];
-    if (updates instanceof Map) {
-      columns = Array.from(updates.entries());
-    } else {
-      columns = Object.entries(updates);
-    }
-    await this.inner.update(onlyIf, columns);
-  }
-
+  ): Promise<void>;
  /** Count the total number of rows in the dataset. */
-  async countRows(filter?: string): Promise<number> {
-    return await this.inner.countRows(filter);
-  }
-
+  abstract countRows(filter?: string): Promise<number>;
  /** Delete the rows that satisfy the predicate. */
-  async delete(predicate: string): Promise<void> {
-    await this.inner.delete(predicate);
-  }
-
+  abstract delete(predicate: string): Promise<void>;
  /**
   * Create an index to speed up queries.
   *
@@ -206,6 +160,9 @@ export class Table {
   * Indices on vector columns will speed up vector searches.
   * Indices on scalar columns will speed up filtering (in both
   * vector and non-vector searches)
+   *
+   * @note We currently don't support custom named indexes,
+   * The index name will always be `${column}_idx`
   * @example
   * // If the column has a vector (fixed size list) data type then
   * // an IvfPq vector index will be created.
@@ -225,13 +182,10 @@ export class Table {
   * // Or create a Scalar index
   * await table.createIndex("my_float_col");
   */
-  async createIndex(column: string, options?: Partial<IndexOptions>) {
-    // Bit of a hack to get around the fact that TS has no package-scope.
-    // biome-ignore lint/suspicious/noExplicitAny: skip
-    const nativeIndex = (options?.config as any)?.inner;
-    await this.inner.createIndex(nativeIndex, column, options?.replace);
-  }
-
+  abstract createIndex(
+    column: string,
+    options?: Partial<IndexOptions>,
+  ): Promise<void>;
  /**
   * Create a {@link Query} Builder.
   *
@@ -282,44 +236,20 @@ export class Table {
   * }
   * @returns {Query} A builder that can be used to parameterize the query
   */
-  query(): Query {
-    return new Query(this.inner);
-  }
-
+  abstract query(): Query;
  /**
   * Create a search query to find the nearest neighbors
   * of the given query vector
   * @param {string} query - the query. This will be converted to a vector using the table's provided embedding function
   * @rejects {Error} If no embedding functions are defined in the table
   */
-  search(query: string): Promise<VectorQuery>;
+  abstract search(query: string): Promise<VectorQuery>;
  /**
   * Create a search query to find the nearest neighbors
   * of the given query vector
   * @param {IntoVector} query - the query vector
   */
-  search(query: IntoVector): VectorQuery;
-  search(query: string | IntoVector): Promise<VectorQuery> | VectorQuery {
-    if (typeof query !== "string") {
-      return this.vectorSearch(query);
-    } else {
-      return this.#getEmbeddingFunctions().then(async (functions) => {
-        // TODO: Support multiple embedding functions
-        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
-          .values()
-          .next().value;
-        if (!embeddingFunc) {
-          return Promise.reject(
-            new Error("No embedding functions are defined in the table"),
-          );
-        }
-        const embeddings =
-          await embeddingFunc.function.computeQueryEmbeddings(query);
-        return this.query().nearestTo(embeddings);
-      });
-    }
-  }
-
+  abstract search(query: IntoVector): VectorQuery;
  /**
   * Search the table with a given query vector.
   *
@@ -327,11 +257,7 @@ export class Table {
   * is the same thing as calling `nearestTo` on the builder returned
   * by `query`.  @see {@link Query#nearestTo} for more details.
   */
-  vectorSearch(vector: IntoVector): VectorQuery {
-    return this.query().nearestTo(vector);
-  }
-
-  // TODO: Support BatchUDF
+  abstract vectorSearch(vector: IntoVector): VectorQuery;
  /**
   * Add new columns with defined values.
   * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
@@ -339,19 +265,14 @@ export class Table {
   * expressions will be evaluated for each row in the table, and can
   * reference existing columns in the table.
   */
-  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
-    await this.inner.addColumns(newColumnTransforms);
-  }
+  abstract addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;

  /**
   * Alter the name or nullability of columns.
   * @param {ColumnAlteration[]} columnAlterations One or more alterations to
   * apply to columns.
   */
-  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
-    await this.inner.alterColumns(columnAlterations);
-  }
-
+  abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
  /**
   * Drop one or more columns from the dataset
   *
@@ -363,15 +284,10 @@ export class Table {
   * be nested column references (e.g. "a.b.c") or top-level column names
   * (e.g. "a").
   */
-  async dropColumns(columnNames: string[]): Promise<void> {
-    await this.inner.dropColumns(columnNames);
-  }
-
+  abstract dropColumns(columnNames: string[]): Promise<void>;
  /** Retrieve the version of the table */
-  async version(): Promise<number> {
-    return await this.inner.version();
-  }

+  abstract version(): Promise<number>;
  /**
   * Checks out a specific version of the table _This is an in-place operation._
   *
@@ -397,19 +313,14 @@ export class Table {
   * console.log(await table.version()); // 2
   * ```
   */
-  async checkout(version: number): Promise<void> {
-    await this.inner.checkout(version);
-  }
-
+  abstract checkout(version: number): Promise<void>;
  /**
   * Checkout the latest version of the table. _This is an in-place operation._
   *
   * The table will be set back into standard mode, and will track the latest
   * version of the table.
   */
-  async checkoutLatest(): Promise<void> {
-    await this.inner.checkoutLatest();
-  }
+  abstract checkoutLatest(): Promise<void>;

  /**
   * Restore the table to the currently checked out version
@@ -423,10 +334,7 @@ export class Table {
   * Once the operation concludes the table will no longer be in a checked
   * out state and the read_consistency_interval, if any, will apply.
   */
-  async restore(): Promise<void> {
-    await this.inner.restore();
-  }
-
+  abstract restore(): Promise<void>;
  /**
   * Optimize the on-disk data and indices for better performance.
   *
@@ -457,6 +365,200 @@ export class Table {
   *  you have added or modified 100,000 or more records or run more than 20 data
   *  modification operations.
   */
+  abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
+  /** List all indices that have been created with {@link Table.createIndex} */
+  abstract listIndices(): Promise<IndexConfig[]>;
+  /** Return the table as an arrow table */
+  abstract toArrow(): Promise<ArrowTable>;
+
+  abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
+
+  /** List all the stats of a specified index
+   *
+   * @param {string} name The name of the index.
+   * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
+   */
+  abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+
+  static async parseTableData(
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    data: Record<string, unknown>[] | ArrowTable<any>,
+    options?: Partial<CreateTableOptions>,
+    streaming = false,
+  ) {
+    let mode: string = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+
+    if (mode === "create" && existOk) {
+      mode = "exist_ok";
+    }
+
+    let table: ArrowTable;
+    if (isArrowTable(data)) {
+      table = data;
+    } else {
+      table = makeArrowTable(data, options);
+    }
+    if (streaming) {
+      const buf = await fromTableToStreamBuffer(
+        table,
+        options?.embeddingFunction,
+        options?.schema,
+      );
+      return { buf, mode };
+    } else {
+      const buf = await fromTableToBuffer(
+        table,
+        options?.embeddingFunction,
+        options?.schema,
+      );
+      return { buf, mode };
+    }
+  }
+}
+
+export class LocalTable extends Table {
+  private readonly inner: _NativeTable;
+
+  constructor(inner: _NativeTable) {
+    super();
+    this.inner = inner;
+  }
+  get name(): string {
+    return this.inner.name;
+  }
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  close(): void {
+    this.inner.close();
+  }
+
+  display(): string {
+    return this.inner.display();
+  }
+
+  private async getEmbeddingFunctions(): Promise<
+    Map<string, EmbeddingFunctionConfig>
+  > {
+    const schema = await this.schema();
+    const registry = getRegistry();
+    return registry.parseFunctions(schema.metadata);
+  }
+
+  /** Get the schema of the table. */
+  async schema(): Promise<Schema> {
+    const schemaBuf = await this.inner.schema();
+    const tbl = tableFromIPC(schemaBuf);
+    return tbl.schema;
+  }
+
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+    const mode = options?.mode ?? "append";
+    const schema = await this.schema();
+    const registry = getRegistry();
+    const functions = registry.parseFunctions(schema.metadata);
+
+    const buffer = await fromDataToBuffer(
+      data,
+      functions.values().next().value,
+      schema,
+    );
+    await this.inner.add(buffer, mode);
+  }
+
+  async update(
+    updates: Map<string, string> | Record<string, string>,
+    options?: Partial<UpdateOptions>,
+  ) {
+    const onlyIf = options?.where;
+    let columns: [string, string][];
+    if (updates instanceof Map) {
+      columns = Array.from(updates.entries());
+    } else {
+      columns = Object.entries(updates);
+    }
+    await this.inner.update(onlyIf, columns);
+  }
+
+  async countRows(filter?: string): Promise<number> {
+    return await this.inner.countRows(filter);
+  }
+
+  async delete(predicate: string): Promise<void> {
+    await this.inner.delete(predicate);
+  }
+
+  async createIndex(column: string, options?: Partial<IndexOptions>) {
+    // Bit of a hack to get around the fact that TS has no package-scope.
+    // biome-ignore lint/suspicious/noExplicitAny: skip
+    const nativeIndex = (options?.config as any)?.inner;
+    await this.inner.createIndex(nativeIndex, column, options?.replace);
+  }
+
+  query(): Query {
+    return new Query(this.inner);
+  }
+
+  search(query: string): Promise<VectorQuery>;
+
+  search(query: IntoVector): VectorQuery;
+  search(query: string | IntoVector): Promise<VectorQuery> | VectorQuery {
+    if (typeof query !== "string") {
+      return this.vectorSearch(query);
+    } else {
+      return this.getEmbeddingFunctions().then(async (functions) => {
+        // TODO: Support multiple embedding functions
+        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
+          .values()
+          .next().value;
+        if (!embeddingFunc) {
+          return Promise.reject(
+            new Error("No embedding functions are defined in the table"),
+          );
+        }
+        const embeddings =
+          await embeddingFunc.function.computeQueryEmbeddings(query);
+        return this.query().nearestTo(embeddings);
+      });
+    }
+  }
+
+  vectorSearch(vector: IntoVector): VectorQuery {
+    return this.query().nearestTo(vector);
+  }
+
+  // TODO: Support BatchUDF
+
+  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+    await this.inner.addColumns(newColumnTransforms);
+  }
+
+  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+    await this.inner.alterColumns(columnAlterations);
+  }
+
+  async dropColumns(columnNames: string[]): Promise<void> {
+    await this.inner.dropColumns(columnNames);
+  }
+
+  async version(): Promise<number> {
+    return await this.inner.version();
+  }
+
+  async checkout(version: number): Promise<void> {
+    await this.inner.checkout(version);
+  }
+
+  async checkoutLatest(): Promise<void> {
+    await this.inner.checkoutLatest();
+  }
+
+  async restore(): Promise<void> {
+    await this.inner.restore();
+  }
+
  async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
    let cleanupOlderThanMs;
    if (
@@ -469,13 +571,23 @@ export class Table {
    return await this.inner.optimize(cleanupOlderThanMs);
  }

-  /** List all indices that have been created with {@link Table.createIndex} */
  async listIndices(): Promise<IndexConfig[]> {
    return await this.inner.listIndices();
  }

-  /** Return the table as an arrow table */
  async toArrow(): Promise<ArrowTable> {
    return await this.query().toArrow();
  }
+
+  async indexStats(name: string): Promise<IndexStatistics | undefined> {
+    const stats = await this.inner.indexStats(name);
+    if (stats === null) {
+      return undefined;
+    }
+    return stats;
+  }
+  mergeInsert(on: string | string[]): MergeInsertBuilder {
+    on = Array.isArray(on) ? on : [on];
+    return new MergeInsertBuilder(this.inner.mergeInsert(on));
+  }
 }
--- a/nodejs/lancedb/util.ts
+++ b/nodejs/lancedb/util.ts
@@ -0,0 +1,35 @@
+export class TTLCache {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  private readonly cache: Map<string, { value: any; expires: number }>;
+
+  /**
+   * @param ttl Time to live in milliseconds
+   */
+  constructor(private readonly ttl: number) {
+    this.cache = new Map();
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  get(key: string): any | undefined {
+    const entry = this.cache.get(key);
+    if (entry === undefined) {
+      return undefined;
+    }
+
+    if (entry.expires < Date.now()) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    return entry.value;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  set(key: string, value: any): void {
+    this.cache.set(key, { value, expires: Date.now() + this.ttl });
+  }
+
+  delete(key: string): void {
+    this.cache.delete(key);
+  }
+}
--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
--- a/nodejs/npm/darwin-x64/package.json
+++ b/nodejs/npm/darwin-x64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.5.1",
+	"version": "0.6.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.5.0",
+  "version": "0.5.2",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.5.0",
+      "version": "0.5.2",
      "cpu": [
        "x64",
        "arm64"
@@ -19,6 +19,7 @@
      ],
      "dependencies": {
        "apache-arrow": "^15.0.0",
+        "axios": "^1.7.2",
        "openai": "^4.29.2",
        "reflect-metadata": "^0.2.2"
      },
@@ -28,6 +29,7 @@
        "@biomejs/biome": "^1.7.3",
        "@jest/globals": "^29.7.0",
        "@napi-rs/cli": "^2.18.0",
+        "@types/axios": "^0.14.0",
        "@types/jest": "^29.1.2",
        "@types/tmp": "^0.2.6",
        "apache-arrow-old": "npm:apache-arrow@13.0.0",
@@ -3123,6 +3125,16 @@
        "tslib": "^2.4.0"
      }
    },
+    "node_modules/@types/axios": {
+      "version": "0.14.0",
+      "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
+      "integrity": "sha512-KqQnQbdYE54D7oa/UmYVMZKq7CO4l8DEENzOKc4aBRwxCXSlJXGz83flFx5L7AWrOQnmuN3kVsRdt+GZPPjiVQ==",
+      "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!",
+      "dev": true,
+      "dependencies": {
+        "axios": "*"
+      }
+    },
    "node_modules/@types/babel__core": {
      "version": "7.20.5",
      "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz",
@@ -3497,6 +3509,16 @@
      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
    },
+    "node_modules/axios": {
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz",
+      "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==",
+      "dependencies": {
+        "follow-redirects": "^1.15.6",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
    "node_modules/babel-jest": {
      "version": "29.7.0",
      "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz",
@@ -4478,6 +4500,25 @@
      "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==",
      "dev": true
    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/form-data": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
@@ -6359,6 +6400,11 @@
        "node": ">= 6"
      }
    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
    "node_modules/punycode": {
      "version": "2.3.1",
      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -1,6 +1,16 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.5.1",
+  "description": "LanceDB: A serverless, low-latency vector database for AI applications",
+  "keywords": [
+    "database",
+    "lance",
+    "lancedb",
+    "search",
+    "vector",
+    "vector database",
+    "ann"
+  ],
+  "version": "0.6.0",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -38,7 +48,8 @@
    "typedoc": "^0.25.7",
    "typedoc-plugin-markdown": "^3.17.1",
    "typescript": "^5.3.3",
-    "typescript-eslint": "^7.1.0"
+    "typescript-eslint": "^7.1.0",
+    "@types/axios": "^0.14.0"
  },
  "ava": {
    "timeout": "3m"
@@ -66,6 +77,7 @@
  },
  "dependencies": {
    "apache-arrow": "^15.0.0",
+    "axios": "^1.7.2",
    "openai": "^4.29.2",
    "reflect-metadata": "^0.2.2"
  }
--- a/nodejs/src/connection.rs
+++ b/nodejs/src/connection.rs
@@ -56,12 +56,6 @@ impl Connection {
    #[napi(factory)]
    pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
        let mut builder = ConnectBuilder::new(&uri);
-        if let Some(api_key) = options.api_key {
-            builder = builder.api_key(&api_key);
-        }
-        if let Some(host_override) = options.host_override {
-            builder = builder.host_override(&host_override);
-        }
        if let Some(interval) = options.read_consistency_interval {
            builder =
                builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
--- a/nodejs/src/lib.rs
+++ b/nodejs/src/lib.rs
@@ -20,6 +20,7 @@ mod connection;
 mod error;
 mod index;
 mod iterator;
+pub mod merge;
 mod query;
 mod table;
 mod util;
@@ -27,8 +28,6 @@ mod util;
 #[napi(object)]
 #[derive(Debug)]
 pub struct ConnectionOptions {
-    pub api_key: Option<String>,
-    pub host_override: Option<String>,
    /// (For LanceDB OSS only): The interval, in seconds, at which to check for
    /// updates to the table from other processes. If None, then consistency is not
    /// checked. For performance reasons, this is the default. For strong
--- a/nodejs/src/merge.rs
+++ b/nodejs/src/merge.rs
@@ -0,0 +1,53 @@
+use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+
+#[napi]
+#[derive(Clone)]
+/// A builder used to create and run a merge insert operation
+pub struct NativeMergeInsertBuilder {
+    pub(crate) inner: MergeInsertBuilder,
+}
+
+#[napi]
+impl NativeMergeInsertBuilder {
+    #[napi]
+    pub fn when_matched_update_all(&self, condition: Option<String>) -> Self {
+        let mut this = self.clone();
+        this.inner.when_matched_update_all(condition);
+        this
+    }
+
+    #[napi]
+    pub fn when_not_matched_insert_all(&self) -> Self {
+        let mut this = self.clone();
+        this.inner.when_not_matched_insert_all();
+        this
+    }
+    #[napi]
+    pub fn when_not_matched_by_source_delete(&self, filter: Option<String>) -> Self {
+        let mut this = self.clone();
+        this.inner.when_not_matched_by_source_delete(filter);
+        this
+    }
+
+    #[napi]
+    pub async fn execute(&self, buf: Buffer) -> napi::Result<()> {
+        let data = ipc_file_to_batches(buf.to_vec())
+            .and_then(IntoArrow::into_arrow)
+            .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+
+        let this = self.clone();
+
+        this.inner
+            .execute(data)
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("Failed to execute merge insert: {}", e)))
+    }
+}
+
+impl From<MergeInsertBuilder> for NativeMergeInsertBuilder {
+    fn from(inner: MergeInsertBuilder) -> Self {
+        Self { inner }
+    }
+}
--- a/nodejs/src/table.rs
+++ b/nodejs/src/table.rs
@@ -23,13 +23,14 @@ use napi_derive::napi;

 use crate::error::NapiErrorExt;
 use crate::index::Index;
+use crate::merge::NativeMergeInsertBuilder;
 use crate::query::{Query, VectorQuery};

 #[napi]
 pub struct Table {
    // We keep a duplicate of the table name so we can use it for error
    // messages even if the table has been closed
-    name: String,
+    pub name: String,
    pub(crate) inner: Option<LanceDbTable>,
 }

@@ -328,16 +329,31 @@ impl Table {
            .map(IndexConfig::from)
            .collect::<Vec<_>>())
    }
+
+    #[napi]
+    pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
+        let tbl = self.inner_ref()?.as_native().unwrap();
+        let stats = tbl.index_stats(&index_name).await.default_error()?;
+        Ok(stats.map(IndexStatistics::from))
+    }
+
+    #[napi]
+    pub fn merge_insert(&self, on: Vec<String>) -> napi::Result<NativeMergeInsertBuilder> {
+        let on: Vec<_> = on.iter().map(String::as_str).collect();
+        Ok(self.inner_ref()?.merge_insert(on.as_slice()).into())
+    }
 }

 #[napi(object)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {
+    /// The name of the index
+    pub name: String,
    /// The type of the index
    pub index_type: String,
    /// The columns in the index
    ///
-    /// Currently this is always an array of size 1.  In the future there may
+    /// Currently this is always an array of size 1. In the future there may
    /// be more columns to represent composite indices.
    pub columns: Vec<String>,
 }
@@ -348,6 +364,7 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
        Self {
            index_type,
            columns: value.columns,
+            name: value.name,
        }
    }
 }
@@ -430,3 +447,40 @@ pub struct AddColumnsSql {
    /// The expression can reference other columns in the table.
    pub value_sql: String,
 }
+
+#[napi(object)]
+pub struct IndexStatistics {
+    /// The number of rows indexed by the index
+    pub num_indexed_rows: f64,
+    /// The number of rows not indexed
+    pub num_unindexed_rows: f64,
+    /// The type of the index
+    pub index_type: Option<String>,
+    /// The metadata for each index
+    pub indices: Vec<IndexMetadata>,
+}
+impl From<lancedb::index::IndexStatistics> for IndexStatistics {
+    fn from(value: lancedb::index::IndexStatistics) -> Self {
+        Self {
+            num_indexed_rows: value.num_indexed_rows as f64,
+            num_unindexed_rows: value.num_unindexed_rows as f64,
+            index_type: value.index_type.map(|t| format!("{:?}", t)),
+            indices: value.indices.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+#[napi(object)]
+pub struct IndexMetadata {
+    pub metric_type: Option<String>,
+    pub index_type: Option<String>,
+}
+
+impl From<lancedb::index::IndexMetadata> for IndexMetadata {
+    fn from(value: lancedb::index::IndexMetadata) -> Self {
+        Self {
+            metric_type: value.metric_type,
+            index_type: value.index_type,
+        }
+    }
+}
--- a/python/.bumpversion.toml
+++ b/python/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.8.2"
+current_version = "0.9.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.8.2"
+version = "0.9.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -3,7 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
    "deprecation",
-    "pylance==0.12.1",
+    "pylance==0.13.0",
    "ratelimiter~=1.0",
    "requests>=2.31.0",
    "retry>=0.9.2",
@@ -57,15 +57,10 @@ tests = [
    "duckdb",
    "pytz",
    "polars>=0.19",
-    "tantivy"
+    "tantivy",
 ]
 dev = ["ruff", "pre-commit"]
-docs = [
-    "mkdocs",
-    "mkdocs-jupyter",
-    "mkdocs-material",
-    "mkdocstrings[python]",
-]
+docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
 embeddings = [
    "openai>=1.6.1",
@@ -100,5 +95,5 @@ addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
 markers = [
    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
    "asyncio",
-    "s3_test"
+    "s3_test",
 ]
--- a/python/python/lancedb/fts.py
+++ b/python/python/lancedb/fts.py
@@ -29,7 +29,10 @@ from .table import LanceTable


 def create_index(
-    index_path: str, text_fields: List[str], ordering_fields: List[str] = None
+    index_path: str,
+    text_fields: List[str],
+    ordering_fields: List[str] = None,
+    tokenizer_name: str = "default",
 ) -> tantivy.Index:
    """
    Create a new Index (not populated)
@@ -42,6 +45,8 @@ def create_index(
        List of text fields to index
    ordering_fields: List[str]
        List of unsigned type fields to order by at search time
+    tokenizer_name : str, default "default"
+        The tokenizer to use

    Returns
    -------
@@ -56,7 +61,7 @@ def create_index(
    schema_builder.add_integer_field("doc_id", stored=True)
    # data fields
    for name in text_fields:
-        schema_builder.add_text_field(name, stored=True)
+        schema_builder.add_text_field(name, stored=True, tokenizer_name=tokenizer_name)
    if ordering_fields:
        for name in ordering_fields:
            schema_builder.add_unsigned_field(name, fast=True)
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -337,7 +337,6 @@ class Table(ABC):
        For example, the following scan will be faster if the column ``my_col`` has
        a scalar index:

-        .. code-block:: python

            import lancedb

@@ -348,8 +347,6 @@ class Table(ABC):
        Scalar indices can also speed up scans containing a vector search and a
        prefilter:

-        .. code-block::python
-
            import lancedb

            db = lancedb.connect("/data/lance")
@@ -385,7 +382,6 @@ class Table(ABC):
        Examples
        --------

-        .. code-block:: python

            import lance

@@ -1175,6 +1171,7 @@ class LanceTable(Table):
        *,
        replace: bool = False,
        writer_heap_size: Optional[int] = 1024 * 1024 * 1024,
+        tokenizer_name: str = "default",
    ):
        """Create a full-text search index on the table.

@@ -1193,6 +1190,10 @@ class LanceTable(Table):
        ordering_field_names:
            A list of unsigned type fields to index to optionally order
            results on at search time
+        tokenizer_name: str, default "default"
+            The tokenizer to use for the index. Can be "raw", "default" or the 2 letter
+            language code followed by "_stem". So for english it would be "en_stem".
+            For available languages see: https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html
        """
        from .fts import create_index, populate_index

@@ -1218,6 +1219,7 @@ class LanceTable(Table):
            self._get_fts_index_path(),
            field_names,
            ordering_fields=ordering_field_names,
+            tokenizer_name=tokenizer_name,
        )
        populate_index(
            index,
--- a/python/python/tests/docs/test_embeddings_optional.py
+++ b/python/python/tests/docs/test_embeddings_optional.py
@@ -0,0 +1,27 @@
+import lancedb
+
+# --8<-- [start:imports]
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+# --8<-- [end:imports]
+import pytest
+
+
+@pytest.mark.slow
+def test_embeddings_openai():
+    # --8<-- [start:openai_embeddings]
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add([{"text": "hello world"}, {"text": "goodbye world"}])
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    # --8<-- [end:openai_embeddings]
--- a/python/python/tests/test_fts.py
+++ b/python/python/tests/test_fts.py
@@ -66,6 +66,17 @@ def test_create_index(tmp_path):
    assert os.path.exists(str(tmp_path / "index"))


+def test_create_index_with_stemming(tmp_path, table):
+    index = ldb.fts.create_index(
+        str(tmp_path / "index"), ["text"], tokenizer_name="en_stem"
+    )
+    assert isinstance(index, tantivy.Index)
+    assert os.path.exists(str(tmp_path / "index"))
+
+    # Check stemming by running tokenizer on non empty table
+    table.create_fts_index("text", tokenizer_name="en_stem")
+
+
 def test_populate_index(tmp_path, table):
    index = ldb.fts.create_index(str(tmp_path / "index"), ["text"])
    assert ldb.fts.populate_index(index, table, ["text"]) == len(table)
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -0,0 +1,2 @@
+[toolchain]
+channel = "1.79.0"
--- a/rust/ffi/node/Cargo.toml
+++ b/rust/ffi/node/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.5.1"
+version = "0.6.0"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true
--- a/rust/ffi/node/src/table.rs
+++ b/rust/ffi/node/src/table.rs
@@ -463,6 +463,7 @@ impl JsTable {
        Ok(promise)
    }

+    #[allow(deprecated)]
    pub(crate) fn js_index_stats(mut cx: FunctionContext) -> JsResult<JsPromise> {
        let js_table = cx.this().downcast_or_throw::<JsBox<Self>, _>(&mut cx)?;
        let rt = runtime(&mut cx)?;
--- a/rust/lancedb/Cargo.toml
+++ b/rust/lancedb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.5.1"
+version = "0.6.0"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
--- a/rust/lancedb/src/index.rs
+++ b/rust/lancedb/src/index.rs
@@ -80,6 +80,8 @@ pub enum IndexType {

 /// A description of an index currently configured on a column
 pub struct IndexConfig {
+    /// The name of the index
+    pub name: String,
    /// The type of the index
    pub index_type: IndexType,
    /// The columns in the index
--- a/rust/lancedb/src/polars_arrow_convertors.rs
+++ b/rust/lancedb/src/polars_arrow_convertors.rs
@@ -84,7 +84,8 @@ pub fn convert_polars_arrow_array_to_arrow_rs_array(
    arrow_datatype: arrow_schema::DataType,
 ) -> std::result::Result<arrow_array::ArrayRef, arrow_schema::ArrowError> {
    let polars_c_array = polars_arrow::ffi::export_array_to_c(polars_array);
-    let arrow_c_array = unsafe { mem::transmute(polars_c_array) };
+    // Safety: `polars_arrow::ffi::ArrowArray` has the same memory layout as `arrow::ffi::FFI_ArrowArray`.
+    let arrow_c_array: arrow_data::ffi::FFI_ArrowArray = unsafe { mem::transmute(polars_c_array) };
    Ok(arrow_array::make_array(unsafe {
        arrow::ffi::from_ffi_and_data_type(arrow_c_array, arrow_datatype)
    }?))
@@ -96,7 +97,8 @@ fn convert_arrow_rs_array_to_polars_arrow_array(
    polars_arrow_dtype: polars::datatypes::ArrowDataType,
 ) -> Result<Box<dyn polars_arrow::array::Array>> {
    let arrow_c_array = arrow::ffi::FFI_ArrowArray::new(&arrow_rs_array.to_data());
-    let polars_c_array = unsafe { mem::transmute(arrow_c_array) };
+    // Safety: `polars_arrow::ffi::ArrowArray` has the same memory layout as `arrow::ffi::FFI_ArrowArray`.
+    let polars_c_array: polars_arrow::ffi::ArrowArray = unsafe { mem::transmute(arrow_c_array) };
    Ok(unsafe { polars_arrow::ffi::import_array_from_c(polars_c_array, polars_arrow_dtype) }?)
 }

@@ -104,7 +106,9 @@ fn convert_polars_arrow_field_to_arrow_rs_field(
    polars_arrow_field: polars_arrow::datatypes::Field,
 ) -> Result<arrow_schema::Field> {
    let polars_c_schema = polars_arrow::ffi::export_field_to_c(&polars_arrow_field);
-    let arrow_c_schema: arrow::ffi::FFI_ArrowSchema = unsafe { mem::transmute(polars_c_schema) };
+    // Safety: `polars_arrow::ffi::ArrowSchema` has the same memory layout as `arrow::ffi::FFI_ArrowSchema`.
+    let arrow_c_schema: arrow::ffi::FFI_ArrowSchema =
+        unsafe { mem::transmute::<_, _>(polars_c_schema) };
    let arrow_rs_dtype = arrow_schema::DataType::try_from(&arrow_c_schema)?;
    Ok(arrow_schema::Field::new(
        polars_arrow_field.name,
@@ -118,6 +122,8 @@ fn convert_arrow_rs_field_to_polars_arrow_field(
 ) -> Result<polars_arrow::datatypes::Field> {
    let arrow_rs_dtype = arrow_rs_field.data_type();
    let arrow_c_schema = arrow::ffi::FFI_ArrowSchema::try_from(arrow_rs_dtype)?;
-    let polars_c_schema: polars_arrow::ffi::ArrowSchema = unsafe { mem::transmute(arrow_c_schema) };
+    // Safety: `polars_arrow::ffi::ArrowSchema` has the same memory layout as `arrow::ffi::FFI_ArrowSchema`.
+    let polars_c_schema: polars_arrow::ffi::ArrowSchema =
+        unsafe { mem::transmute::<_, _>(arrow_c_schema) };
    Ok(unsafe { polars_arrow::ffi::import_field_from_c(&polars_c_schema) }?)
 }
--- a/rust/lancedb/src/table.rs
+++ b/rust/lancedb/src/table.rs
@@ -1206,28 +1206,36 @@ impl NativeTable {
            .await)
    }

+    #[deprecated(since = "0.5.2", note = "Please use `index_stats` instead")]
    pub async fn count_indexed_rows(&self, index_uuid: &str) -> Result<Option<usize>> {
+        #[allow(deprecated)]
        match self.load_index_stats(index_uuid).await? {
            Some(stats) => Ok(Some(stats.num_indexed_rows)),
            None => Ok(None),
        }
    }

+    #[deprecated(since = "0.5.2", note = "Please use `index_stats` instead")]
    pub async fn count_unindexed_rows(&self, index_uuid: &str) -> Result<Option<usize>> {
+        #[allow(deprecated)]
        match self.load_index_stats(index_uuid).await? {
            Some(stats) => Ok(Some(stats.num_unindexed_rows)),
            None => Ok(None),
        }
    }

+    #[deprecated(since = "0.5.2", note = "Please use `index_stats` instead")]
    pub async fn get_index_type(&self, index_uuid: &str) -> Result<Option<String>> {
+        #[allow(deprecated)]
        match self.load_index_stats(index_uuid).await? {
            Some(stats) => Ok(Some(stats.index_type.unwrap_or_default())),
            None => Ok(None),
        }
    }

+    #[deprecated(since = "0.5.2", note = "Please use `index_stats` instead")]
    pub async fn get_distance_type(&self, index_uuid: &str) -> Result<Option<String>> {
+        #[allow(deprecated)]
        match self.load_index_stats(index_uuid).await? {
            Some(stats) => Ok(Some(
                stats
@@ -1240,16 +1248,8 @@ impl NativeTable {
        }
    }

-    pub async fn load_indices(&self) -> Result<Vec<VectorIndex>> {
-        let dataset = self.dataset.get().await?;
-        let (indices, mf) = futures::try_join!(dataset.load_indices(), dataset.latest_manifest())?;
-        Ok(indices
-            .iter()
-            .map(|i| VectorIndex::new_from_format(&mf, i))
-            .collect())
-    }
-
-    async fn load_index_stats(&self, index_uuid: &str) -> Result<Option<IndexStatistics>> {
+    #[deprecated(since = "0.5.2", note = "Please use `index_stats` instead")]
+    pub async fn load_index_stats(&self, index_uuid: &str) -> Result<Option<IndexStatistics>> {
        let index = self
            .load_indices()
            .await?
@@ -1268,6 +1268,35 @@ impl NativeTable {
        Ok(Some(index_stats))
    }

+    /// Get statistics about an index.
+    /// Returns an error if the index does not exist.
+    pub async fn index_stats<S: AsRef<str>>(
+        &self,
+        index_name: S,
+    ) -> Result<Option<IndexStatistics>> {
+        self.dataset
+            .get()
+            .await?
+            .index_statistics(index_name.as_ref())
+            .await
+            .ok()
+            .map(|stats| {
+                serde_json::from_str(&stats).map_err(|e| Error::InvalidInput {
+                    message: format!("error deserializing index statistics: {}", e),
+                })
+            })
+            .transpose()
+    }
+
+    pub async fn load_indices(&self) -> Result<Vec<VectorIndex>> {
+        let dataset = self.dataset.get().await?;
+        let (indices, mf) = futures::try_join!(dataset.load_indices(), dataset.latest_manifest())?;
+        Ok(indices
+            .iter()
+            .map(|i| VectorIndex::new_from_format(&mf, i))
+            .collect())
+    }
+
    async fn create_ivf_pq_index(
        &self,
        index: IvfPqIndexBuilder,
@@ -1860,12 +1889,20 @@ impl TableInternal for NativeTable {
                }
                columns.push(field.name.clone());
            }
-            Ok(IndexConfig { index_type: if is_vector { crate::index::IndexType::IvfPq } else { crate::index::IndexType::BTree }, columns })
+            let index_type = if is_vector {
+                crate::index::IndexType::IvfPq
+            } else {
+                crate::index::IndexType::BTree
+            };
+
+            let name = idx.name.clone();
+            Ok(IndexConfig { index_type, columns, name })
        }).collect::<Result<Vec<_>>>()
    }
 }

 #[cfg(test)]
+#[allow(deprecated)]
 mod tests {
    use std::iter;
    use std::sync::atomic::{AtomicBool, Ordering};
--- a/rust/lancedb/src/table/merge.rs
+++ b/rust/lancedb/src/table/merge.rs
@@ -23,6 +23,7 @@ use super::TableInternal;
 /// A builder used to create and run a merge insert operation
 ///
 /// See [`super::Table::merge_insert`] for more context
+#[derive(Debug, Clone)]
 pub struct MergeInsertBuilder {
    table: Arc<dyn TableInternal>,
    pub(super) on: Vec<String>,
Author	SHA1	Message	Date
Lance Release	5ce5f64da3	Bump version: 0.6.0-beta.0 → 0.6.0	2024-06-25 00:25:45 +00:00
Lance Release	c582b0fc63	Bump version: 0.5.2 → 0.6.0-beta.0	2024-06-25 00:25:45 +00:00
Lance Release	bc0814767b	Bump version: 0.9.0-beta.0 → 0.9.0	2024-06-25 00:25:27 +00:00
Lance Release	8960a8e535	Bump version: 0.8.2 → 0.9.0-beta.0	2024-06-25 00:25:27 +00:00
Weston Pace	a8568ddc72	feat: upgrade to lance 0.13.0 (#1404 )	2024-06-24 17:22:57 -07:00
Cory Grinstead	55f88346d0	feat(nodejs): table.indexStats (#1361 ) closes https://github.com/lancedb/lancedb/issues/1359	2024-06-21 17:06:52 -05:00
Will Jones	dfb9a28795	ci(node): add description and keywords for lancedb package (#1398 )	2024-06-21 14:43:35 -07:00
Cory Grinstead	a797f5fe59	feat(nodejs): feature parity [5/N] - add `query.filter()` alias (#1391 ) to make the transition from `vectordb` to `@lancedb/lancedb` as seamless as possible, this adds `query.filter` with a deprecated tag. depends on https://github.com/lancedb/lancedb/pull/1390 see actual diff here https://github.com/universalmind303/lancedb/compare/list-indices-name...universalmind303:query-filter	2024-06-21 16:03:58 -05:00
Cory Grinstead	3cd84c9375	feat(nodejs): feature parity [4/N] - add 'name' to 'IndexConfig' for 'listIndices' (#1390 ) depends on https://github.com/lancedb/lancedb/pull/1386 see actual diff here https://github.com/universalmind303/lancedb/compare/create-table-args...universalmind303:list-indices-name	2024-06-21 15:45:02 -05:00
Cory Grinstead	5ca83fdc99	fix(node): node build (#1396 ) i have no idea why this fixes the build.	2024-06-21 15:42:22 -05:00
Cory Grinstead	33cc9b682f	feat(nodejs): feature parity [3/N] - `createTable({name, data, ...options})` (#1386 ) adds support for the `vectordb` syntax of `createTable({name, data, ...options})`. depends on https://github.com/lancedb/lancedb/pull/1380 see actual diff here https://github.com/universalmind303/lancedb/compare/table-name...universalmind303:create-table-args	2024-06-21 12:17:39 -05:00
Cory Grinstead	b3e5ac6d2a	feat(nodejs): feature parity [2/N] - add `table.name` and `lancedb.connect({args})` (#1380 ) depends on https://github.com/lancedb/lancedb/pull/1378 see proper diff here https://github.com/universalmind303/lancedb/compare/remote-table-node...universalmind303:lancedb:table-name	2024-06-21 11:38:26 -05:00
josca42	0fe844034d	feat: enable stemming (#1356 ) Added the ability to specify tokenizer_name, when creating a full text search index using tantivy. This enables the use of language specific stemming. Also updated the [guide on full text search](https://lancedb.github.io/lancedb/fts/) with a short section on choosing tokenizer. Fixes #1315	2024-06-20 14:23:55 -07:00
Cory Grinstead	f41eb899dc	chore(rust): lock toolchain & fix clippy (#1389 ) - fix some clippy errors from ci running a different toolchain. - add some saftey notes about some unsafe blocks. - locks the toolchain so that it is consistent across dev and CI.	2024-06-20 12:13:03 -05:00
Cory Grinstead	e7022b990e	feat(nodejs): feature parity [1/N] - remote table (#1378 ) closes https://github.com/lancedb/lancedb/issues/1362	2024-06-17 15:23:27 -05:00
Weston Pace	ea86dad4b7	feat: upgrade lance to 0.12.2-beta.2 (#1381 )	2024-06-14 05:43:26 -07:00
harsha-mangena	a45656b8b6	docs: remove code-block:: python from docs (#1366 ) - refer #1264 - fixed minor documentation issue	2024-06-11 13:13:02 -07:00
Cory Grinstead	bc19a75f65	feat(nodejs): merge insert (#1351 ) closes https://github.com/lancedb/lancedb/issues/1349	2024-06-11 15:05:15 -05:00
Ryan Green	8e348ab4bd	fix: use JS naming convention in new index stats fields (#1377 ) Changes new index stats fields in node client from snake case to camel case.	2024-06-10 16:41:31 -02:30
Raghav Dixit	96914a619b	docs: llama-index integration (#1347 ) Updated api refrence and usage for llama index integration.	2024-06-09 23:52:18 +05:30
Beinan	3c62806b6a	fix(java): the JVM crash when using jdk 8 (#1372 ) The Optional::isEmpty does not exist in java 8, so we should use isPresent instead	2024-06-08 22:43:41 -07:00
Ayush Chaurasia	72f339a0b3	docs: add note about embedding api not being available on cloud (#1371 )	2024-06-09 03:57:23 +05:30
QianZhu	b9e3cfbdca	fix: add status to remote listIndices return (#1364 ) expose `status` returned by remote listIndices	2024-06-08 09:52:35 -07:00
Ayush Chaurasia	5e30648f45	docs: fix example path (#1367 )	2024-06-07 19:40:50 -07:00
Ayush Chaurasia	76fc16c7a1	docs: add retriever guide, address minor onboarding feedbacks & enhancement (#1326 ) - Tried to address some onboarding feedbacks listed in https://github.com/lancedb/lancedb/issues/1224 - Improve visibility of pydantic integration and embedding API. (Based on onboarding feedback - Many ways of ingesting data, defining schema but not sure what to use in a specific use-case) - Add a guide that takes users through testing and improving retriever performance using built-in utilities like hybrid-search and reranking - Add some benchmarks for the above - Add missing cohere docs --------- Co-authored-by: Weston Pace <weston.pace@gmail.com>	2024-06-08 06:25:31 +05:30
Weston Pace	007f9c1af8	chore: change build machine for linux arm (#1360 )	2024-06-06 13:22:58 -07:00
Lance Release	27e4ad3f11	Updating package-lock.json	2024-06-05 13:47:44 +00:00
Lance Release	df42943ccf	Bump version: 0.5.2-beta.0 → 0.5.2	2024-06-05 13:47:28 +00:00
Lance Release	3eec9ea740	Bump version: 0.5.1 → 0.5.2-beta.0	2024-06-05 13:47:27 +00:00