Bump version: 0.13.0-beta.0 → 0.13.0-beta.1

fix(node): update default log level to warn (#1801 )
🤦
2025-12-23 05:19:58 +00:00 · 2024-11-06 17:26:17 +00:00 · 2024-11-06 09:13:53 -08:00 · 2024-11-05 23:21:50 +00:00 · 2024-11-05 23:21:32 +00:00 · 2024-11-05 23:21:13 +00:00
114 changed files with 3051 additions and 1701 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.11.0-beta.1"
+current_version = "0.13.0-beta.1"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,15 +18,18 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
+rust-version = "1.80.0" # TODO: lower this once we upgrade Lance again.

 [workspace.dependencies]
-lance = { "version" = "=0.18.2", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.18.2" }
-lance-linalg = { "version" = "=0.18.2" }
-lance-table = { "version" = "=0.18.2" }
-lance-testing = { "version" = "=0.18.2" }
-lance-datafusion = { "version" = "=0.18.2" }
-lance-encoding = { "version" = "=0.18.2" }
+lance = { "version" = "=0.19.2", "features" = [
+    "dynamodb",
+], git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-index = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-linalg = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-table = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-testing = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-datafusion = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
+lance-encoding = { "version" = "=0.19.2", git = "https://github.com/lancedb/lance.git", tag = "v0.19.2-beta.3" }
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"
@@ -40,6 +43,7 @@ async-trait = "0"
 chrono = "0.4.35"
 datafusion-common = "41.0"
 datafusion-physical-plan = "41.0"
+env_logger = "0.10"
 half = { "version" = "=2.4.1", default-features = false, features = [
    "num-traits",
 ] }
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -90,6 +90,9 @@ markdown_extensions:
  - pymdownx.emoji:
      emoji_index: !!python/name:material.extensions.emoji.twemoji
      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+  - markdown.extensions.toc:
+      baselevel: 1
+      permalink: ""

 nav:
  - Home:
@@ -97,7 +100,7 @@ nav:
      - 🏃🏼‍♂️ Quick start: basic.md
      - 📚 Concepts:
          - Vector search: concepts/vector_search.md
-          - Indexing: 
+          - Indexing:
            - IVFPQ: concepts/index_ivfpq.md
            - HNSW: concepts/index_hnsw.md
          - Storage: concepts/storage.md
@@ -106,7 +109,8 @@ nav:
          - Working with tables: guides/tables.md
          - Building a vector index: ann_indexes.md
          - Vector Search: search.md
-          - Full-text search: fts.md
+          - Full-text search (native): fts.md
+          - Full-text search (tantivy-based): fts_tantivy.md
          - Building a scalar index: guides/scalar_index.md
          - Hybrid search:
              - Overview: hybrid_search/hybrid_search.md
@@ -145,10 +149,10 @@ nav:
              - Reranking: guides/tuning_retrievers/2_reranking.md
              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
      - 🧬 Managing embeddings:
-          - Understand Embeddings: embeddings/understanding_embeddings.md 
+          - Understand Embeddings: embeddings/understanding_embeddings.md
          - Get Started: embeddings/index.md
          - Embedding functions: embeddings/embedding_functions.md
-          - Available models: 
+          - Available models:
              - Overview: embeddings/default_embedding_functions.md
              - Text Embedding Functions:
                  - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
@@ -197,7 +201,7 @@ nav:
              - Evaluation: examples/python_examples/evaluations.md
              - AI Agent: examples/python_examples/aiagent.md
              - Recommender System: examples/python_examples/recommendersystem.md
-              - Miscellaneous: 
+              - Miscellaneous:
                  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
                  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
          - 👾 JavaScript:
@@ -207,9 +211,10 @@ nav:
              - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
          - 🦀 Rust:
              - Overview: examples/examples_rust.md
-      - Studies:
+      - 📓 Studies:
          - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
      - 💭 FAQs: faq.md
+      - 🔍 Troubleshooting: troubleshooting.md
      - ⚙️ API reference:
          - 🐍 Python: python/python.md
          - 👾 JavaScript (vectordb): javascript/modules.md
@@ -225,7 +230,7 @@ nav:
  - Quick start: basic.md
  - Concepts:
      - Vector search: concepts/vector_search.md
-      - Indexing: 
+      - Indexing:
        - IVFPQ: concepts/index_ivfpq.md
        - HNSW: concepts/index_hnsw.md
      - Storage: concepts/storage.md
@@ -234,7 +239,8 @@ nav:
      - Working with tables: guides/tables.md
      - Building an ANN index: ann_indexes.md
      - Vector Search: search.md
-      - Full-text search: fts.md
+      - Full-text search (native): fts.md
+      - Full-text search (tantivy-based): fts_tantivy.md
      - Building a scalar index: guides/scalar_index.md
      - Hybrid search:
          - Overview: hybrid_search/hybrid_search.md
@@ -273,10 +279,10 @@ nav:
          - Reranking: guides/tuning_retrievers/2_reranking.md
          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
  - Managing Embeddings:
-      - Understand Embeddings: embeddings/understanding_embeddings.md 
+      - Understand Embeddings: embeddings/understanding_embeddings.md
      - Get Started: embeddings/index.md
      - Embedding functions: embeddings/embedding_functions.md
-      - Available models: 
+      - Available models:
          - Overview: embeddings/default_embedding_functions.md
          - Text Embedding Functions:
              - Sentence Transformers: embeddings/available_embedding_models/text_embedding_functions/sentence_transformers.md
@@ -321,7 +327,7 @@ nav:
          - Evaluation: examples/python_examples/evaluations.md
          - AI Agent: examples/python_examples/aiagent.md
          - Recommender System: examples/python_examples/recommendersystem.md
-          - Miscellaneous: 
+          - Miscellaneous:
              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
      - 👾 JavaScript:
@@ -364,5 +370,4 @@ extra:
    - icon: fontawesome/brands/x-twitter
      link: https://twitter.com/lancedb
    - icon: fontawesome/brands/linkedin
-      link: https://www.linkedin.com/company/lancedb      
-      
+      link: https://www.linkedin.com/company/lancedb
--- a/docs/src/examples/python_examples/chatbot.md
+++ b/docs/src/examples/python_examples/chatbot.md
@@ -36,6 +36,6 @@
 [aware_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/chatbot_using_Llama2_&_lanceDB/main.ipynb
 [aware_ghost]: https://blog.lancedb.com/context-aware-chatbot-using-llama-2-lancedb-as-vector-database-4d771d95c755

-[csv_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Chat_with_csv_file
-[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Chat_with_csv_file/main.ipynb
+[csv_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Chat_with_csv_file
+[csv_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Chat_with_csv_file/main.ipynb
 [csv_ghost]: https://blog.lancedb.com/p/d8c71df4-e55f-479a-819e-cde13354a6a3/
--- a/docs/src/examples/python_examples/multimodal.md
+++ b/docs/src/examples/python_examples/multimodal.md
@@ -12,7 +12,7 @@ LanceDB supports multimodal search by indexing and querying vector representatio
 |:----------------|:-----------------|:-----------|
 | **Multimodal CLIP: DiffusionDB 🌐💥**    | Multi-Modal Search with **CLIP** and **LanceDB** Using **DiffusionDB** Data for Combined Text and Image Understanding ! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
 | **Multimodal CLIP: Youtube Videos 📹👀** | Search **Youtube videos** using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
-| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
+| **Multimodal Image + Text Search 📸🔍**               | Find **relevant documents** and **images** with a single query using **LanceDB's** multimodal search capabilities, to seamlessly integrate text and visuals ! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
 | **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Learn how **Cambrian-1** works, using an example of **Vision-Centric** exploration on images found through vector search ! Work on **Flickr-8k** dataset 🔎                        | [![Kaggle](https://img.shields.io/badge/Kaggle-035a7d?style=for-the-badge&logo=kaggle&logoColor=white)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |


--- a/docs/src/examples/python_examples/rag.md
+++ b/docs/src/examples/python_examples/rag.md
@@ -70,12 +70,12 @@ Build RAG (Retrieval-Augmented Generation) with  LanceDB, a powerful solution fo
 [flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
 [flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/

-[query_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker
-[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker/main.ipynb
+[query_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/QueryExpansion%26Reranker
+[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/QueryExpansion&Reranker/main.ipynb


-[fusion_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion
-[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion/main.ipynb
+[fusion_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/RAG_Fusion
+[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/RAG_Fusion/main.ipynb

 [agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
 [agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb
--- a/docs/src/examples/python_examples/recommendersystem.md
+++ b/docs/src/examples/python_examples/recommendersystem.md
@@ -19,8 +19,8 @@ Deliver personalized experiences with Recommender Systems. 🎁
 [movie_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.py


-[genre_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/movie-recommendation-with-genres
-[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
+[genre_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/movie-recommendation-with-genres
+[genre_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/movie-recommendation-with-genres/movie_recommendation_with_doc2vec_and_lancedb.ipynb
 [genre_ghost]: https://blog.lancedb.com/movie-recommendation-system-using-lancedb-and-doc2vec/

 [product_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/product-recommender
@@ -33,5 +33,5 @@ Deliver personalized experiences with Recommender Systems. 🎁
 [arxiv_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/arxiv-recommender/main.py
 

-[food_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Food_recommendation
-[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Food_recommendation/main.ipynb
+[food_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/Food_recommendation
+[food_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/Food_recommendation/main.ipynb
--- a/docs/src/examples/python_examples/vector_search.md
+++ b/docs/src/examples/python_examples/vector_search.md
@@ -37,16 +37,16 @@ LanceDB implements vector search algorithms for efficient document retrieval and
 [NER_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/NER-powered-Semantic-Search/NER_powered_Semantic_Search_with_LanceDB.ipynb
 [NER_ghost]: https://blog.lancedb.com/ner-powered-semantic-search-using-lancedb-51051dc3e493

-[audio_search_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/audio_search
-[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb
-[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.py
+[audio_search_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/audio_search
+[audio_search_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.ipynb
+[audio_search_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/audio_search/main.py

-[mls_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa
-[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa/main.ipynb
-[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multi-lingual-wiki-qa/main.py
+[mls_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/multi-lingual-wiki-qa
+[mls_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.ipynb
+[mls_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/archived_examples/multi-lingual-wiki-qa/main.py

-[fr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/facial_recognition
-[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/facial_recognition/main.ipynb
+[fr_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/facial_recognition
+[fr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/facial_recognition/main.ipynb

 [sentiment_analysis_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews
 [sentiment_analysis_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Sentiment-Analysis-Analyse-Hotel-Reviews/Sentiment_Analysis_using_LanceDB.ipynb
@@ -70,8 +70,8 @@ LanceDB implements vector search algorithms for efficient document retrieval and
 [openvino_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Accelerate-Vector-Search-Applications-Using-OpenVINO/clip_text_image_search.ipynb
 [openvino_ghost]: https://blog.lancedb.com/accelerate-vector-search-applications-using-openvino-lancedb/

-[zsic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/zero-shot-image-classification
-[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/zero-shot-image-classification/main.ipynb
+[zsic_github]: https://github.com/lancedb/vectordb-recipes/tree/main/examples/archived_examples/zero-shot-image-classification
+[zsic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/archived_examples/zero-shot-image-classification/main.ipynb
 [zsic_ghost]: https://blog.lancedb.com/zero-shot-image-classification-with-vector-search/


--- a/docs/src/fts.md
+++ b/docs/src/fts.md
@@ -1,21 +1,9 @@
-# Full-text search
+# Full-text search (Native FTS)

-LanceDB provides support for full-text search via Lance (before via [Tantivy](https://github.com/quickwit-oss/tantivy) (Python only)), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
-
-Currently, the Lance full text search is missing some features that are in the Tantivy full text search. This includes query parser and customizing the tokenizer. Thus, in Python, Tantivy is still the default way to do full text search and many of the instructions below apply just to Tantivy-based indices.
-
-
-## Installation (Only for Tantivy-based FTS)
+LanceDB provides support for full-text search via Lance, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.

 !!! note
-    No need to install the tantivy dependency if using native FTS
-
-To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
-
-```sh
-# Say you want to use tantivy==0.20.1
-pip install tantivy==0.20.1
-```
+    The Python SDK uses tantivy-based FTS by default, need to pass `use_tantivy=False` to use native FTS.

 ## Example

@@ -39,7 +27,7 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex

    # passing `use_tantivy=False` to use lance FTS index
    # `use_tantivy=True` by default
-    table.create_fts_index("text")
+    table.create_fts_index("text", use_tantivy=False)
    table.search("puppy").limit(10).select(["text"]).to_list()
    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
    # ...
@@ -93,51 +81,40 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
    ```

 It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
-For now, this is supported in tantivy way only.

-Passing `fts_columns="text"` if you want to specify the columns to search, but it's not available for Tantivy-based full text search.
+Passing `fts_columns="text"` if you want to specify the columns to search.

 !!! note
    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.

 ## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+By default the text is tokenized by splitting on punctuation and whitespaces, and would filter out words that are with length greater than 40, and lowercase all words.

-For now, only the Tantivy-based FTS index supports to specify the tokenizer, so it's only available in Python with `use_tantivy=True`.
+Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.

-=== "use_tantivy=True"
-
-    ```python
-    table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
-    ```
-
-=== "use_tantivy=False"
-
-    [**Not supported yet**](https://github.com/lancedb/lance/issues/1195)
+For example, to enable stemming for English:
+```python
+table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
+```

 the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.

-## Index multiple columns
+The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.

-If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
-
-=== "use_tantivy=True"
-
-    ```python
-    table.create_fts_index(["text1", "text2"])
-    ```
-
-=== "use_tantivy=False"
-
-    [**Not supported yet**](https://github.com/lancedb/lance/issues/1195)
-
-Note that the search API call does not change - you can search over all indexed columns at once.
+For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
+```python
+table.create_fts_index("text",
+                        use_tantivy=False,
+                        language="French",
+                        stem=True,
+                        ascii_folding=True)
+```

 ## Filtering

-Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
-applied on top of the full text search results. This can be invoked via the familiar
-`where` syntax:
+LanceDB full text search supports to filter the search results by a condition, both pre-filtering and post-filtering are supported.
+
+This can be invoked via the familiar `where` syntax:

 === "Python"

@@ -169,98 +146,17 @@ applied on top of the full text search results. This can be invoked via the fami
        .await?;
    ```

-## Sorting
-
-!!! warning "Warn"
-    Sorting is available for only Tantivy-based FTS
-
-You can pre-sort the documents by specifying `ordering_field_names` when
-creating the full-text search index. Once pre-sorted, you can then specify
-`ordering_field_name` while searching to return results sorted by the given
-field. For example,
-
-```python
-table.create_fts_index(["text_field"], use_tantivy=True, ordering_field_names=["sort_by_field"])
-
-(table.search("terms", ordering_field_name="sort_by_field")
- .limit(20)
- .to_list())
-```
-
-!!! note
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see
-    an error during indexing that looks like
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
 ## Phrase queries vs. terms queries

 !!! warning "Warn"
    Lance-based FTS doesn't support queries using boolean operators `OR`, `AND`.

 For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+or a **terms** search query like `old man sea`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).

-!!! tip "Note"
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
-
-    ```py
-    # This raises a syntax error
-    table.search("they could have been dogs OR cats")
-    ```
-
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
-    the query is treated as a phrase query.
-
-    ```py
-    # This works!
-    table.search("they could have been dogs or cats")
-    ```
-
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
-
-
-## Configurations (Only for Tantivy-based FTS)
-
-By default, LanceDB configures a 1GB heap size limit for creating the index. You can
-reduce this if running on a smaller node, or increase this for faster performance while
-indexing a larger corpus.
-
+To search for a phrase, the index must be created with `with_position=True`:
 ```python
-# configure a 512MB heap size
-heap = 1024 * 1024 * 512
-table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
+table.create_fts_index("text", use_tantivy=False, with_position=True)
 ```
-
-## Current limitations
-
-For that Tantivy-based FTS:
-
-1. Currently we do not yet support incremental writes.
-   If you add data after FTS index creation, it won't be reflected
-   in search results until you do a full reindex.
-
-2. We currently only support local filesystem paths for the FTS index.
-   This is a tantivy limitation. We've implemented an object store plugin
-   but there's no way in tantivy-py to specify to use it.
+This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
--- a/docs/src/fts_tantivy.md
+++ b/docs/src/fts_tantivy.md
@@ -0,0 +1,162 @@
+# Full-text search (Tantivy-based FTS)
+
+LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+
+The tantivy-based FTS is only available in Python and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
+
+## Installation
+
+To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
+
+```sh
+# Say you want to use tantivy==0.20.1
+pip install tantivy==0.20.1
+```
+
+## Example
+
+Consider that we have a LanceDB table named `my_table`, whose string column `content` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table(
+    "my_table",
+    data=[
+        {"id": 1, "vector": [3.1, 4.1], "title": "happy puppy", "content": "Frodo was a happy puppy", "meta": "foo"},
+        {"id": 2, "vector": [5.9, 26.5], "title": "playing kittens", "content": "There are several kittens playing around the puppy", "meta": "bar"},
+    ],
+)
+
+# passing `use_tantivy=False` to use lance FTS index
+# `use_tantivy=True` by default
+table.create_fts_index("content", use_tantivy=True)
+table.search("puppy").limit(10).select(["content"]).to_list()
+# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+# ...
+```
+
+It would search on all indexed columns by default, so it's useful when there are multiple indexed columns.
+
+!!! note
+    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
+
+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("content", use_tantivy=True, tokenizer_name="en_stem", replace=True)
+```
+
+the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+## Index multiple columns
+
+If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
+
+```python
+table.create_fts_index(["title", "content"], use_tantivy=True, replace=True)
+```
+
+Note that the search API call does not change - you can search over all indexed columns at once.
+
+## Filtering
+
+Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+applied on top of the full text search results (see [native FTS](fts.md) if you need pre-filtering). This can be invoked via the familiar
+`where` syntax:
+
+```python
+table.search("puppy").limit(10).where("meta='foo'").to_list()
+```
+
+## Sorting
+
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example,
+
+```python
+table.create_fts_index(["content"], use_tantivy=True, ordering_field_names=["id"], replace=True)
+
+(table.search("puppy", ordering_field_name="id")
+ .limit(20)
+ .to_list())
+```
+
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see
+    an error during indexing that looks like
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
+## Phrase queries vs. terms queries
+
+For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
+
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
+    ```
+
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
+
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
+    ```
+
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+1. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["title", "content"], use_tantivy=True, writer_heap_size=heap, replace=True)
+```
+
+## Current limitations
+
+1. Currently we do not yet support incremental writes.
+   If you add data after FTS index creation, it won't be reflected
+   in search results until you do a full reindex.
+
+2. We currently only support local filesystem paths for the FTS index.
+   This is a tantivy limitation. We've implemented an object store plugin
+   but there's no way in tantivy-py to specify to use it.
--- a/docs/src/guides/storage.md
+++ b/docs/src/guides/storage.md
@@ -498,7 +498,7 @@ This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` envir

 #### S3 Express

-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
+LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional infrastructure configuration for the compute service, such as EC2 or Lambda. Please refer to [Networking requirements for S3 Express One Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-networking.html).

 To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.

--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -49,7 +49,8 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Working with tables](guides/tables.md): Learn how to work with tables and their associated functions
 * [Indexing](ann_indexes.md): Understand how to create indexes
 * [Vector search](search.md): Learn how to perform vector similarity search
-* [Full-text search](fts.md): Learn how to perform full-text search
+* [Full-text search (native)](fts.md): Learn how to perform full-text search
+* [Full-text search (tantivy-based)](fts_tantivy.md): Learn how to perform full-text search using Tantivy
 * [Managing embeddings](embeddings/index.md): Managing embeddings and the embedding functions API in LanceDB
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
--- a/docs/src/integrations/langchain.md
+++ b/docs/src/integrations/langchain.md
@@ -1,5 +1,10 @@
-# Langchain
-![Illustration](../assets/langchain.png)
+**LangChain** is a framework designed for building applications with large language models (LLMs) by chaining together various components. It supports a range of functionalities including memory, agents, and chat models, enabling developers to create context-aware applications.
+
+![Illustration](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/langchain_rag.png)
+
+LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities.
+
+Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs.

 ## Quick Start
 You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
@@ -26,20 +31,28 @@ print(docs[0].page_content)

 ## Documentation
 In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
+
 You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  

-The exhaustive list of parameters for `LanceDB` vector store are :  
- `connection`: (Optional) `lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.  
- `embedding`: Langchain embedding model.  
- `vector_key`: (Optional) Column name to use for vector's in the table. Defaults to `'vector'`.   
- `id_key`: (Optional) Column name to use for id's in the table. Defaults to `'id'`.  
- `text_key`: (Optional) Column name to use for text in the table. Defaults to `'text'`.  
- `table_name`: (Optional) Name of your table in the database. Defaults to `'vectorstore'`.  
- `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
- `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
- `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
- `reranker`: (Optional) The reranker to use for LanceDB.
- `relevance_score_fn`: (Optional[Callable[[float], float]]) Langchain relevance score function to be used. Defaults to `None`. 
+The exhaustive list of parameters for `LanceDB` vector store are : 
+
+|Name|type|Purpose|default|
+|:----|:----|:----|:----|
+|`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.|`None`|
+|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.|
+|`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`|
+|`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`|
+|`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`|
+|`text_key` |(Optional) `str` |Column name to use for text in the table.|`'text'`|
+|`table_name` |(Optional) `str`| Name of your table in the database.|`'vectorstore'`|
+|`api_key` |(Optional `str`) |API key to use for LanceDB cloud database.|`None`|
+|`region` |(Optional) `str`| Region to use for LanceDB cloud database.|Only for LanceDB Cloud : `None`.|
+|`mode` |(Optional) `str` |Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`|
+|`table`| (Optional) `Any`|You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`|
+|`distance`|(Optional) `str`|The choice of distance metric used to calculate the similarity between vectors.|`'l2'`|
+|`reranker` |(Optional) `Any`|The reranker to use for LanceDB.|`None`|
+|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`|
+|`limit`|`int`|Set the maximum number of results to return.|`DEFAULT_K` (it is 4)|

 ```python
 db_url = "db://lang_test" # url of db you created
@@ -51,19 +64,24 @@ vector_store = LanceDB(
    api_key=api_key, #(dont include for local API)
    region=region, #(dont include for local API)
    embedding=embeddings,
-    table_name='langchain_test' #Optional
+    table_name='langchain_test' # Optional
    )
 ```

 ### Methods 

 ##### add_texts()
- `texts`: `Iterable` of strings to add to the vectorstore.
- `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
- `ids`: Optional `list` of ids to associate with the texts. 
- `kwargs`: `Any`

-This method adds texts and stores respective embeddings automatically.
+This method turn texts into embedding and add it to the database.
+
+|Name|Purpose|defaults|
+|:---|:---|:---|
+|`texts`|`Iterable` of strings to add to the vectorstore.|Provided by user|
+|`metadatas`|Optional `list[dict()]` of metadatas associated with the texts.|`None`|
+|`ids`|Optional `list` of ids to associate with the texts.|`None`|
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
+It returns list of ids of the added texts.

 ```python
 vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
@@ -78,14 +96,25 @@ pd_df.to_csv("docsearch.csv", index=False)
 # you can also create a new vector store object using an older connection object:
 vector_store = LanceDB(connection=tbl, embedding=embeddings)
 ```
-##### create_index() 
- `col_name`: `Optional[str] = None`
- `vector_col`: `Optional[str] = None`
- `num_partitions`: `Optional[int] = 256`
- `num_sub_vectors`: `Optional[int] = 96`
- `index_cache_size`: `Optional[int] = None`

-This method creates an index for the vector store. For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+------
+
+
+##### create_index() 
+
+This method creates a scalar(for non-vector cols) or a vector index on a table.
+
+|Name|type|Purpose|defaults|
+|:---|:---|:---|:---|
+|`vector_col`|`Optional[str]`| Provide if you want to create index on a vector column. |`None`|
+|`col_name`|`Optional[str]`| Provide if you want to create index on a non-vector column. |`None`|
+|`metric`|`Optional[str]` |Provide the metric to use for vector index. choice of metrics: 'L2', 'dot', 'cosine'. |`L2`|
+|`num_partitions`|`Optional[int]`|Number of partitions to use for the index.|`256`|
+|`num_sub_vectors`|`Optional[int]` |Number of sub-vectors to use for the index.|`96`|
+|`index_cache_size`|`Optional[int]` |Size of the index cache.|`None`|
+|`name`|`Optional[str]` |Name of the table to create index on.|`None`|
+
+For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.

 ```python
 # for creating vector index
@@ -96,42 +125,63 @@ vector_store.create_index(col_name='text')

 ```

-##### similarity_search()
- `query`: `str`
- `k`: `Optional[int] = None`
- `filter`: `Optional[Dict[str, str]] = None`
- `fts`: `Optional[bool] = False`
- `name`: `Optional[str] = None`
- `kwargs`: `Any`
+------

-Return documents most similar to the query without relevance scores
+##### similarity_search()
+
+This method performs similarity search based on **text query**.
+
+| Name    | Type                 | Purpose | Default |
+|---------|----------------------|---------|---------|
+| `query` | `str`                |  A `str` representing the text query that you want to search for in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `fts`   | `Optional[bool]`     |   It indicates whether to perform a full-text search (FTS).      | `False`   |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+Return documents most similar to the query **without relevance scores**.

 ```python
 docs = docsearch.similarity_search(query)
 print(docs[0].page_content)
 ```

-##### similarity_search_by_vector()
- `embedding`: `List[float]`
- `k`: `Optional[int] = None`
- `filter`: `Optional[Dict[str, str]] = None`
- `name`: `Optional[str] = None`
- `kwargs`: `Any`
+------

-Returns documents most similar to the query vector.
+##### similarity_search_by_vector()
+
+The method returns documents that are most similar to the specified **embedding (query) vector**. 
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.         | N/A     |
+| `k`         | `Optional[int]`           | It specifies the number of documents to return.        | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`| It is used to filter the search results by specific metadata criteria.        | `None`    |
+| `name`      | `Optional[str]`           | It is used for specifying the name of the table to query. If not provided, it uses the default table set during the initialization of the LanceDB instance.        | `None`    |
+| `kwargs`    | `Any`                     | Other keyworded arguments provided by the user.        | N/A     |
+
+**It does not provide relevance scores.**

 ```python
 docs = docsearch.similarity_search_by_vector(query)
 print(docs[0].page_content)
 ```

-##### similarity_search_with_score()
- `query`: `str`
- `k`: `Optional[int] = None`
- `filter`: `Optional[Dict[str, str]] = None`
- `kwargs`: `Any`
+------

-Returns documents most similar to the query string with relevance scores, gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+##### similarity_search_with_score()
+
+Returns documents most similar to the **query string** along with their relevance scores.
+
+| Name     | Type                      | Purpose | Default |
+|----------|---------------------------|---------|---------|
+| `query`  | `str`                     |A `str` representing the text query you want to search for in the vector store. This query will be converted into an embedding using the specified embedding function.         | N/A     |
+| `k`      | `Optional[int]`           | It specifies the number of documents to return.  | `None`    |
+| `filter` | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria. This allows you to narrow down the search results based on certain metadata attributes associated with the documents.       | `None`    |
+| `kwargs` | `Any`                     |  Other keyworded arguments provided by the user.       | N/A     |
+
+It gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.

 ```python
 docs = docsearch.similarity_search_with_relevance_scores(query)
@@ -139,15 +189,21 @@ print("relevance score - ", docs[0][1])
 print("text- ", docs[0][0].page_content[:1000])
 ```

-##### similarity_search_by_vector_with_relevance_scores()
- `embedding`: `List[float]`
- `k`: `Optional[int] = None`
- `filter`: `Optional[Dict[str, str]] = None`
- `name`: `Optional[str] = None`
- `kwargs`: `Any`
+------

-Return documents most similar to the query vector with relevance scores.
-Relevance score 
+##### similarity_search_by_vector_with_relevance_scores()
+
+Similarity search using **query vector**.
+
+| Name        | Type                      | Purpose | Default |
+|-------------|---------------------------|---------|---------|
+| `embedding` | `List[float]`             | The embedding vector you want to use to search for similar documents in the vector store.        | N/A     |
+| `k`         | `Optional[int]`           |   It specifies the number of documents to return.       | `None`    |
+| `filter`    | `Optional[Dict[str, str]]`|  It is used to filter the search results by specific metadata criteria.       | `None`    |
+| `name`      | `Optional[str]`           |   It is used for specifying the name of the table to query.       | `None`    |
+| `kwargs`    | `Any`                     |   Other keyworded arguments provided by the user.      | N/A     |
+
+The method returns documents most similar to the specified embedding (query) vector, along with their relevance scores.

 ```python
 docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
@@ -155,20 +211,22 @@ print("relevance score - ", docs[0][1])
 print("text- ", docs[0][0].page_content[:1000])
 ```

-##### max_marginal_relevance_search()
- `query`: `str`
- `k`: `Optional[int] = None`
- `fetch_k` : Number of Documents to fetch to pass to MMR algorithm, `Optional[int] = None`
- `lambda_mult`: Number between 0 and 1 that determines the degree
-                        of diversity among the results with 0 corresponding
-                        to maximum diversity and 1 to minimum diversity.
-                        Defaults to 0.5. `float = 0.5`
- `filter`: `Optional[Dict[str, str]] = None`
- `kwargs`: `Any`
+------

-Returns docs selected using the maximal marginal relevance(MMR).
+##### max_marginal_relevance_search()
+
+This method returns docs selected using the maximal marginal relevance(MMR).
 Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.

+| Name | Type | Purpose | Default |
+|---------------|-----------------|-----------|---------|
+| `query`       | `str` | Text to look up documents similar to. | N/A     |
+| `k`   | `Optional[int]` | Number of Documents to return.| `4`       |
+| `fetch_k`| `Optional[int]`| Number of Documents to fetch to pass to MMR algorithm.| `None`    |
+| `lambda_mult` | `float`                   | Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. | `0.5`     |
+| `filter`| `Optional[Dict[str, str]]`| Filter by metadata. | `None`    |
+|`kwargs`| Other keyworded arguments provided by the user. |-|
+
 Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 

 ```python
@@ -186,12 +244,19 @@ result_texts = [doc.page_content for doc in result]
 print(result_texts)
 ```

-##### add_images()
- `uris` : File path to the image. `List[str]`.
- `metadatas` : Optional list of metadatas. `(Optional[List[dict]], optional)`
- `ids` : Optional list of IDs. `(Optional[List[str]], optional)`
+------

-Adds images by automatically creating their embeddings and adds them to the vectorstore.
+##### add_images()
+
+This method ddds images by automatically creating their embeddings and adds them to the vectorstore.
+
+| Name       | Type                          | Purpose                        | Default |
+|------------|-------------------------------|--------------------------------|---------|
+| `uris`     | `List[str]`                   | File path to the image         | N/A     |
+| `metadatas`| `Optional[List[dict]]`        | Optional list of metadatas     | `None`    |
+| `ids`      | `Optional[List[str]]`         | Optional list of IDs           | `None`    |
+
+It returns list of IDs of the added images.

 ```python
 vec_store.add_images(uris=image_uris) 
--- a/docs/src/integrations/phidata.md
+++ b/docs/src/integrations/phidata.md
@@ -45,7 +45,7 @@ Let's see how using LanceDB inside phidata helps in making LLM more useful:

 **Install the following packages in the virtual environment**
 ```python
-pip install lancedb phidata youtube_transcript_api openai ollama pandas numpy
+pip install lancedb phidata youtube_transcript_api openai ollama numpy pandas
 ```

 **Create python files and import necessary libraries**
--- a/docs/src/javascript/README.md
+++ b/docs/src/javascript/README.md
@@ -41,7 +41,6 @@ To build everything fresh:

 ```bash
 npm install
-npm run tsc
 npm run build
 ```

@@ -51,18 +50,6 @@ Then you should be able to run the tests with:
 npm test
 ```

-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
 ### Fix lints

 To run the linter and have it automatically fix all errors
--- a/docs/src/javascript/classes/DefaultWriteOptions.md
+++ b/docs/src/javascript/classes/DefaultWriteOptions.md
@@ -38,4 +38,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation

 #### Defined in

-[index.ts:1019](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1019)
+[index.ts:1359](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1359)
--- a/docs/src/javascript/classes/LocalConnection.md
+++ b/docs/src/javascript/classes/LocalConnection.md
@@ -30,6 +30,7 @@ A connection to a LanceDB database.
 - [dropTable](LocalConnection.md#droptable)
 - [openTable](LocalConnection.md#opentable)
 - [tableNames](LocalConnection.md#tablenames)
+- [withMiddleware](LocalConnection.md#withmiddleware)

 ## Constructors

@@ -46,7 +47,7 @@ A connection to a LanceDB database.

 #### Defined in

-[index.ts:489](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L489)
+[index.ts:739](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L739)

 ## Properties

@@ -56,7 +57,7 @@ A connection to a LanceDB database.

 #### Defined in

-[index.ts:487](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L487)
+[index.ts:737](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L737)

 ___

@@ -74,7 +75,7 @@ ___

 #### Defined in

-[index.ts:486](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L486)
+[index.ts:736](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L736)

 ## Accessors

@@ -92,7 +93,7 @@ ___

 #### Defined in

-[index.ts:494](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L494)
+[index.ts:744](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L744)

 ## Methods

@@ -113,7 +114,7 @@ Creates a new Table, optionally initializing it with new data.
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
-| `data?` | `Record`\<`string`, `unknown`\>[] |
+| `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
 | `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 | `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |

@@ -127,7 +128,7 @@ Creates a new Table, optionally initializing it with new data.

 #### Defined in

-[index.ts:542](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L542)
+[index.ts:788](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L788)

 ___

@@ -158,7 +159,7 @@ ___

 #### Defined in

-[index.ts:576](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L576)
+[index.ts:822](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L822)

 ___

@@ -184,7 +185,7 @@ Drop an existing table.

 #### Defined in

-[index.ts:630](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L630)
+[index.ts:876](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L876)

 ___

@@ -210,7 +211,7 @@ Open a table in the database.

 #### Defined in

-[index.ts:510](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L510)
+[index.ts:760](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L760)

 ▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>

@@ -239,7 +240,7 @@ Connection.openTable

 #### Defined in

-[index.ts:518](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L518)
+[index.ts:768](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L768)

 ▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>

@@ -266,7 +267,7 @@ Connection.openTable

 #### Defined in

-[index.ts:522](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L522)
+[index.ts:772](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L772)

 ___

@@ -286,4 +287,36 @@ Get the names of all tables in the database.

 #### Defined in

-[index.ts:501](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L501)
+[index.ts:751](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L751)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Connection`](../interfaces/Connection.md)
+
+Instrument the behavior of this Connection with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote Connections.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Connection`](../interfaces/Connection.md)
+
+- this Connection instrumented by the passed middleware
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[withMiddleware](../interfaces/Connection.md#withmiddleware)
+
+#### Defined in
+
+[index.ts:880](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L880)
--- a/docs/src/javascript/classes/LocalTable.md
+++ b/docs/src/javascript/classes/LocalTable.md
@@ -37,6 +37,8 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Methods

 - [add](LocalTable.md#add)
+- [addColumns](LocalTable.md#addcolumns)
+- [alterColumns](LocalTable.md#altercolumns)
 - [checkElectron](LocalTable.md#checkelectron)
 - [cleanupOldVersions](LocalTable.md#cleanupoldversions)
 - [compactFiles](LocalTable.md#compactfiles)
@@ -44,13 +46,16 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [createIndex](LocalTable.md#createindex)
 - [createScalarIndex](LocalTable.md#createscalarindex)
 - [delete](LocalTable.md#delete)
+- [dropColumns](LocalTable.md#dropcolumns)
 - [filter](LocalTable.md#filter)
 - [getSchema](LocalTable.md#getschema)
 - [indexStats](LocalTable.md#indexstats)
 - [listIndices](LocalTable.md#listindices)
+- [mergeInsert](LocalTable.md#mergeinsert)
 - [overwrite](LocalTable.md#overwrite)
 - [search](LocalTable.md#search)
 - [update](LocalTable.md#update)
+- [withMiddleware](LocalTable.md#withmiddleware)

 ## Constructors

@@ -74,7 +79,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector

 #### Defined in

-[index.ts:642](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L642)
+[index.ts:892](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L892)

 • **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)

@@ -95,7 +100,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector

 #### Defined in

-[index.ts:649](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L649)
+[index.ts:899](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L899)

 ## Properties

@@ -105,7 +110,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector

 #### Defined in

-[index.ts:639](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L639)
+[index.ts:889](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L889)

 ___

@@ -115,7 +120,7 @@ ___

 #### Defined in

-[index.ts:638](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L638)
+[index.ts:888](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L888)

 ___

@@ -125,7 +130,7 @@ ___

 #### Defined in

-[index.ts:637](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L637)
+[index.ts:887](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L887)

 ___

@@ -143,7 +148,7 @@ ___

 #### Defined in

-[index.ts:640](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L640)
+[index.ts:890](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L890)

 ___

@@ -153,7 +158,7 @@ ___

 #### Defined in

-[index.ts:636](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L636)
+[index.ts:886](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L886)

 ___

@@ -179,7 +184,7 @@ Creates a filter query to find all rows matching the specified criteria

 #### Defined in

-[index.ts:688](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L688)
+[index.ts:938](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L938)

 ## Accessors

@@ -197,7 +202,7 @@ Creates a filter query to find all rows matching the specified criteria

 #### Defined in

-[index.ts:668](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L668)
+[index.ts:918](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L918)

 ___

@@ -215,7 +220,7 @@ ___

 #### Defined in

-[index.ts:849](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L849)
+[index.ts:1171](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1171)

 ## Methods

@@ -229,7 +234,7 @@ Insert records into this Table.

 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |

 #### Returns

@@ -243,7 +248,59 @@ The number of rows added to the table

 #### Defined in

-[index.ts:696](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L696)
+[index.ts:946](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L946)
+
+___
+
+### addColumns
+
+▸ **addColumns**(`newColumnTransforms`): `Promise`\<`void`\>
+
+Add new columns with defined values.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `newColumnTransforms` | \{ `name`: `string` ; `valueSql`: `string`  }[] | pairs of column names and the SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns in the table. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[addColumns](../interfaces/Table.md#addcolumns)
+
+#### Defined in
+
+[index.ts:1195](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1195)
+
+___
+
+### alterColumns
+
+▸ **alterColumns**(`columnAlterations`): `Promise`\<`void`\>
+
+Alter the name or nullability of columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnAlterations` | [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[] | One or more alterations to apply to columns. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[alterColumns](../interfaces/Table.md#altercolumns)
+
+#### Defined in
+
+[index.ts:1201](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1201)

 ___

@@ -257,7 +314,7 @@ ___

 #### Defined in

-[index.ts:861](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L861)
+[index.ts:1183](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1183)

 ___

@@ -280,7 +337,7 @@ Clean up old versions of the table, freeing disk space.

 #### Defined in

-[index.ts:808](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L808)
+[index.ts:1130](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1130)

 ___

@@ -307,16 +364,22 @@ Metrics about the compaction operation.

 #### Defined in

-[index.ts:831](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L831)
+[index.ts:1153](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1153)

 ___

 ### countRows

-▸ **countRows**(): `Promise`\<`number`\>
+▸ **countRows**(`filter?`): `Promise`\<`number`\>

 Returns the number of rows in this table.

+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `filter?` | `string` |
+
 #### Returns

 `Promise`\<`number`\>
@@ -327,7 +390,7 @@ Returns the number of rows in this table.

 #### Defined in

-[index.ts:749](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L749)
+[index.ts:1021](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1021)

 ___

@@ -357,13 +420,13 @@ VectorIndexParams.

 #### Defined in

-[index.ts:734](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L734)
+[index.ts:1003](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1003)

 ___

 ### createScalarIndex

-▸ **createScalarIndex**(`column`, `replace`): `Promise`\<`void`\>
+▸ **createScalarIndex**(`column`, `replace?`): `Promise`\<`void`\>

 Create a scalar index on this Table for the given column

@@ -372,7 +435,7 @@ Create a scalar index on this Table for the given column
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `column` | `string` | The column to index |
-| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+| `replace?` | `boolean` | If false, fail if an index already exists on the column it is always set to true for remote connections Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |

 #### Returns

@@ -392,7 +455,7 @@ await table.createScalarIndex('my_col')

 #### Defined in

-[index.ts:742](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L742)
+[index.ts:1011](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1011)

 ___

@@ -418,7 +481,38 @@ Delete rows from this table.

 #### Defined in

-[index.ts:758](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L758)
+[index.ts:1030](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1030)
+
+___
+
+### dropColumns
+
+▸ **dropColumns**(`columnNames`): `Promise`\<`void`\>
+
+Drop one or more columns from the dataset
+
+This is a metadata-only operation and does not remove the data from the
+underlying storage. In order to remove the data, you must subsequently
+call ``compact_files`` to rewrite the data without the removed columns and
+then call ``cleanup_files`` to remove the old files.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnNames` | `string`[] | The names of the columns to drop. These can be nested column references (e.g. "a.b.c") or top-level column names (e.g. "a"). |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[dropColumns](../interfaces/Table.md#dropcolumns)
+
+#### Defined in
+
+[index.ts:1205](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1205)

 ___

@@ -438,9 +532,13 @@ Creates a filter query to find all rows matching the specified criteria

 [`Query`](Query.md)\<`T`\>

+#### Implementation of
+
+[Table](../interfaces/Table.md).[filter](../interfaces/Table.md#filter)
+
 #### Defined in

-[index.ts:684](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L684)
+[index.ts:934](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L934)

 ___

@@ -454,13 +552,13 @@ ___

 #### Defined in

-[index.ts:854](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L854)
+[index.ts:1176](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1176)

 ___

 ### indexStats

-▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+▸ **indexStats**(`indexName`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>

 Get statistics about an index.

@@ -468,7 +566,7 @@ Get statistics about an index.

 | Name | Type |
 | :------ | :------ |
-| `indexUuid` | `string` |
+| `indexName` | `string` |

 #### Returns

@@ -480,7 +578,7 @@ Get statistics about an index.

 #### Defined in

-[index.ts:845](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L845)
+[index.ts:1167](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1167)

 ___

@@ -500,7 +598,57 @@ List the indicies on this table.

 #### Defined in

-[index.ts:841](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L841)
+[index.ts:1163](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1163)
+
+___
+
+### mergeInsert
+
+▸ **mergeInsert**(`on`, `data`, `args`): `Promise`\<`void`\>
+
+Runs a "merge insert" operation on the table
+
+This operation can add rows, update rows, and remove rows all in a single
+transaction. It is a very generic tool that can be used to create
+behaviors like "insert if not exists", "update or insert (i.e. upsert)",
+or even replace a portion of existing data with new data (e.g. replace
+all data where month="january")
+
+The merge insert operation works by combining new data from a
+**source table** with existing data in a **target table** by using a
+join.  There are three categories of records.
+
+"Matched" records are records that exist in both the source table and
+the target table. "Not matched" records exist only in the source table
+(e.g. these are new data) "Not matched by source" records exist only
+in the target table (this is old data)
+
+The MergeInsertArgs can be used to customize what should happen for
+each category of data.
+
+Please note that the data may appear to be reordered as part of this
+operation.  This is because updated rows will be deleted from the
+dataset and then reinserted at the end with the new values.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `on` | `string` | a column to join on. This is how records from the source table and target table are matched. |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | the new data to insert |
+| `args` | [`MergeInsertArgs`](../interfaces/MergeInsertArgs.md) | parameters controlling how the operation should behave |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[mergeInsert](../interfaces/Table.md#mergeinsert)
+
+#### Defined in
+
+[index.ts:1065](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1065)

 ___

@@ -514,7 +662,7 @@ Insert records into this Table, replacing its contents.

 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |

 #### Returns

@@ -528,7 +676,7 @@ The number of rows added to the table

 #### Defined in

-[index.ts:716](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L716)
+[index.ts:977](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L977)

 ___

@@ -554,7 +702,7 @@ Creates a search query to find the nearest neighbors of the given search term

 #### Defined in

-[index.ts:676](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L676)
+[index.ts:926](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L926)

 ___

@@ -580,4 +728,36 @@ Update rows in this table.

 #### Defined in

-[index.ts:771](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L771)
+[index.ts:1043](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1043)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Table`](../interfaces/Table.md)\<`T`\>
+
+Instrument the behavior of this Table with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote tables.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Table`](../interfaces/Table.md)\<`T`\>
+
+- this Table instrumented by the passed middleware
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[withMiddleware](../interfaces/Table.md#withmiddleware)
+
+#### Defined in
+
+[index.ts:1209](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1209)
--- a/docs/src/javascript/classes/MakeArrowTableOptions.md
+++ b/docs/src/javascript/classes/MakeArrowTableOptions.md
@@ -0,0 +1,82 @@
+[vectordb](../README.md) / [Exports](../modules.md) / MakeArrowTableOptions
+
+# Class: MakeArrowTableOptions
+
+Options to control the makeArrowTable call.
+
+## Table of contents
+
+### Constructors
+
+- [constructor](MakeArrowTableOptions.md#constructor)
+
+### Properties
+
+- [dictionaryEncodeStrings](MakeArrowTableOptions.md#dictionaryencodestrings)
+- [embeddings](MakeArrowTableOptions.md#embeddings)
+- [schema](MakeArrowTableOptions.md#schema)
+- [vectorColumns](MakeArrowTableOptions.md#vectorcolumns)
+
+## Constructors
+
+### constructor
+
+• **new MakeArrowTableOptions**(`values?`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `values?` | `Partial`\<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)\> |
+
+#### Defined in
+
+[arrow.ts:98](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L98)
+
+## Properties
+
+### dictionaryEncodeStrings
+
+• **dictionaryEncodeStrings**: `boolean` = `false`
+
+If true then string columns will be encoded with dictionary encoding
+
+Set this to true if your string columns tend to repeat the same values
+often.  For more precise control use the `schema` property to specify the
+data type for individual columns.
+
+If `schema` is provided then this property is ignored.
+
+#### Defined in
+
+[arrow.ts:96](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L96)
+
+___
+
+### embeddings
+
+• `Optional` **embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`any`\>
+
+#### Defined in
+
+[arrow.ts:85](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L85)
+
+___
+
+### schema
+
+• `Optional` **schema**: `Schema`\<`any`\>
+
+#### Defined in
+
+[arrow.ts:63](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L63)
+
+___
+
+### vectorColumns
+
+• **vectorColumns**: `Record`\<`string`, `VectorColumnOptions`\>
+
+#### Defined in
+
+[arrow.ts:81](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L81)
--- a/docs/src/javascript/classes/OpenAIEmbeddingFunction.md
+++ b/docs/src/javascript/classes/OpenAIEmbeddingFunction.md
@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv

 #### Defined in

-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:22](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L22)

 ## Properties

@@ -50,17 +50,17 @@ An embedding function that automatically creates vector representation for a giv

 #### Defined in

-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:20](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L20)

 ___

 ### \_openai

-• `Private` `Readonly` **\_openai**: `any`
+• `Private` `Readonly` **\_openai**: `OpenAI`

 #### Defined in

-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L19)

 ___

@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.

 #### Defined in

-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:56](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L56)

 ## Methods

@@ -102,4 +102,4 @@ Creates a vector representation for the given values.

 #### Defined in

-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:43](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/openai.ts#L43)
--- a/docs/src/javascript/classes/Query.md
+++ b/docs/src/javascript/classes/Query.md
@@ -19,6 +19,7 @@ A builder for nearest neighbor queries for LanceDB.
 ### Properties

 - [\_embeddings](Query.md#_embeddings)
+- [\_fastSearch](Query.md#_fastsearch)
 - [\_filter](Query.md#_filter)
 - [\_limit](Query.md#_limit)
 - [\_metricType](Query.md#_metrictype)
@@ -34,6 +35,7 @@ A builder for nearest neighbor queries for LanceDB.
 ### Methods

 - [execute](Query.md#execute)
+- [fastSearch](Query.md#fastsearch)
 - [filter](Query.md#filter)
 - [isElectron](Query.md#iselectron)
 - [limit](Query.md#limit)
@@ -65,7 +67,7 @@ A builder for nearest neighbor queries for LanceDB.

 #### Defined in

-[query.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L38)
+[query.ts:39](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L39)

 ## Properties

@@ -75,7 +77,17 @@ A builder for nearest neighbor queries for LanceDB.

 #### Defined in

-[query.ts:36](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L36)
+[query.ts:37](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L37)
+
+___
+
+### \_fastSearch
+
+• `Private` **\_fastSearch**: `boolean`
+
+#### Defined in
+
+[query.ts:36](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L36)

 ___

@@ -85,7 +97,7 @@ ___

 #### Defined in

-[query.ts:33](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L33)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L33)

 ___

@@ -95,7 +107,7 @@ ___

 #### Defined in

-[query.ts:29](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L29)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L29)

 ___

@@ -105,7 +117,7 @@ ___

 #### Defined in

-[query.ts:34](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L34)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L34)

 ___

@@ -115,7 +127,7 @@ ___

 #### Defined in

-[query.ts:31](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L31)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L31)

 ___

@@ -125,7 +137,7 @@ ___

 #### Defined in

-[query.ts:35](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L35)
+[query.ts:35](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L35)

 ___

@@ -135,7 +147,7 @@ ___

 #### Defined in

-[query.ts:26](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L26)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L26)

 ___

@@ -145,7 +157,7 @@ ___

 #### Defined in

-[query.ts:28](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L28)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L28)

 ___

@@ -155,7 +167,7 @@ ___

 #### Defined in

-[query.ts:30](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L30)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L30)

 ___

@@ -165,7 +177,7 @@ ___

 #### Defined in

-[query.ts:32](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L32)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L32)

 ___

@@ -175,7 +187,7 @@ ___

 #### Defined in

-[query.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L27)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L27)

 ___

@@ -201,7 +213,7 @@ A filter statement to be applied to this query.

 #### Defined in

-[query.ts:87](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L87)
+[query.ts:90](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L90)

 ## Methods

@@ -223,7 +235,30 @@ Execute the query and return the results as an Array of Objects

 #### Defined in

-[query.ts:115](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L115)
+[query.ts:127](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L127)
+
+___
+
+### fastSearch
+
+▸ **fastSearch**(`value`): [`Query`](Query.md)\<`T`\>
+
+Skip searching un-indexed data. This can make search faster, but will miss
+any data that is not yet indexed.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `boolean` |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Defined in
+
+[query.ts:119](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L119)

 ___

@@ -245,7 +280,7 @@ A filter statement to be applied to this query.

 #### Defined in

-[query.ts:82](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L82)
+[query.ts:85](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L85)

 ___

@@ -259,7 +294,7 @@ ___

 #### Defined in

-[query.ts:142](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L142)
+[query.ts:155](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L155)

 ___

@@ -268,6 +303,7 @@ ___
 ▸ **limit**(`value`): [`Query`](Query.md)\<`T`\>

 Sets the number of results that will be returned
+default value is 10

 #### Parameters

@@ -281,7 +317,7 @@ Sets the number of results that will be returned

 #### Defined in

-[query.ts:55](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L55)
+[query.ts:58](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L58)

 ___

@@ -307,7 +343,7 @@ MetricType for the different options

 #### Defined in

-[query.ts:102](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L102)
+[query.ts:105](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L105)

 ___

@@ -329,7 +365,7 @@ The number of probes used. A higher number makes search more accurate but also s

 #### Defined in

-[query.ts:73](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L73)
+[query.ts:76](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L76)

 ___

@@ -349,7 +385,7 @@ ___

 #### Defined in

-[query.ts:107](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L107)
+[query.ts:110](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L110)

 ___

@@ -371,7 +407,7 @@ Refine the results by reading extra elements and re-ranking them in memory.

 #### Defined in

-[query.ts:64](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L64)
+[query.ts:67](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L67)

 ___

@@ -393,4 +429,4 @@ Return only the specified columns.

 #### Defined in

-[query.ts:93](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L93)
+[query.ts:96](https://github.com/lancedb/lancedb/blob/92179835/node/src/query.ts#L96)
--- a/docs/src/javascript/enums/IndexStatus.md
+++ b/docs/src/javascript/enums/IndexStatus.md
@@ -0,0 +1,52 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IndexStatus
+
+# Enumeration: IndexStatus
+
+## Table of contents
+
+### Enumeration Members
+
+- [Done](IndexStatus.md#done)
+- [Failed](IndexStatus.md#failed)
+- [Indexing](IndexStatus.md#indexing)
+- [Pending](IndexStatus.md#pending)
+
+## Enumeration Members
+
+### Done
+
+• **Done** = ``"done"``
+
+#### Defined in
+
+[index.ts:713](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L713)
+
+___
+
+### Failed
+
+• **Failed** = ``"failed"``
+
+#### Defined in
+
+[index.ts:714](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L714)
+
+___
+
+### Indexing
+
+• **Indexing** = ``"indexing"``
+
+#### Defined in
+
+[index.ts:712](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L712)
+
+___
+
+### Pending
+
+• **Pending** = ``"pending"``
+
+#### Defined in
+
+[index.ts:711](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L711)
--- a/docs/src/javascript/enums/MetricType.md
+++ b/docs/src/javascript/enums/MetricType.md
@@ -22,7 +22,7 @@ Cosine distance

 #### Defined in

-[index.ts:1041](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1041)
+[index.ts:1381](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1381)

 ___

@@ -34,7 +34,7 @@ Dot product

 #### Defined in

-[index.ts:1046](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1046)
+[index.ts:1386](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1386)

 ___

@@ -46,4 +46,4 @@ Euclidean distance

 #### Defined in

-[index.ts:1036](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1036)
+[index.ts:1376](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1376)
--- a/docs/src/javascript/enums/WriteMode.md
+++ b/docs/src/javascript/enums/WriteMode.md
@@ -22,7 +22,7 @@ Append new data to the table.

 #### Defined in

-[index.ts:1007](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1007)
+[index.ts:1347](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1347)

 ___

@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).

 #### Defined in

-[index.ts:1003](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1003)
+[index.ts:1343](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1343)

 ___

@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.

 #### Defined in

-[index.ts:1005](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1005)
+[index.ts:1345](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1345)
--- a/docs/src/javascript/interfaces/AwsCredentials.md
+++ b/docs/src/javascript/interfaces/AwsCredentials.md
@@ -18,7 +18,7 @@

 #### Defined in

-[index.ts:54](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L54)
+[index.ts:68](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L68)

 ___

@@ -28,7 +28,7 @@ ___

 #### Defined in

-[index.ts:56](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L56)
+[index.ts:70](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L70)

 ___

@@ -38,4 +38,4 @@ ___

 #### Defined in

-[index.ts:58](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L58)
+[index.ts:72](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L72)
--- a/docs/src/javascript/interfaces/CleanupStats.md
+++ b/docs/src/javascript/interfaces/CleanupStats.md
@@ -19,7 +19,7 @@ The number of bytes removed from disk.

 #### Defined in

-[index.ts:878](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L878)
+[index.ts:1218](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1218)

 ___

@@ -31,4 +31,4 @@ The number of old table versions removed.

 #### Defined in

-[index.ts:882](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L882)
+[index.ts:1222](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1222)
--- a/docs/src/javascript/interfaces/ColumnAlteration.md
+++ b/docs/src/javascript/interfaces/ColumnAlteration.md
@@ -0,0 +1,53 @@
+[vectordb](../README.md) / [Exports](../modules.md) / ColumnAlteration
+
+# Interface: ColumnAlteration
+
+A definition of a column alteration. The alteration changes the column at
+`path` to have the new name `name`, to be nullable if `nullable` is true,
+and to have the data type `data_type`. At least one of `rename` or `nullable`
+must be provided.
+
+## Table of contents
+
+### Properties
+
+- [nullable](ColumnAlteration.md#nullable)
+- [path](ColumnAlteration.md#path)
+- [rename](ColumnAlteration.md#rename)
+
+## Properties
+
+### nullable
+
+• `Optional` **nullable**: `boolean`
+
+Set the new nullability. Note that a nullable column cannot be made non-nullable.
+
+#### Defined in
+
+[index.ts:638](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L638)
+
+___
+
+### path
+
+• **path**: `string`
+
+The path to the column to alter. This is a dot-separated path to the column.
+If it is a top-level column then it is just the name of the column. If it is
+a nested column then it is the path to the column, e.g. "a.b.c" for a column
+`c` nested inside a column `b` nested inside a column `a`.
+
+#### Defined in
+
+[index.ts:633](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L633)
+
+___
+
+### rename
+
+• `Optional` **rename**: `string`
+
+#### Defined in
+
+[index.ts:634](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L634)
--- a/docs/src/javascript/interfaces/CompactionMetrics.md
+++ b/docs/src/javascript/interfaces/CompactionMetrics.md
@@ -22,7 +22,7 @@ fragments added.

 #### Defined in

-[index.ts:933](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L933)
+[index.ts:1273](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1273)

 ___

@@ -35,7 +35,7 @@ file.

 #### Defined in

-[index.ts:928](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L928)
+[index.ts:1268](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1268)

 ___

@@ -47,7 +47,7 @@ The number of new fragments that were created.

 #### Defined in

-[index.ts:923](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L923)
+[index.ts:1263](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1263)

 ___

@@ -59,4 +59,4 @@ The number of fragments that were removed.

 #### Defined in

-[index.ts:919](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L919)
+[index.ts:1259](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1259)
--- a/docs/src/javascript/interfaces/CompactionOptions.md
+++ b/docs/src/javascript/interfaces/CompactionOptions.md
@@ -24,7 +24,7 @@ Default is true.

 #### Defined in

-[index.ts:901](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L901)
+[index.ts:1241](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1241)

 ___

@@ -38,7 +38,7 @@ the deleted rows. Default is 10%.

 #### Defined in

-[index.ts:907](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L907)
+[index.ts:1247](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1247)

 ___

@@ -46,11 +46,11 @@ ___

 • `Optional` **maxRowsPerGroup**: `number`

-The maximum number of rows per group. Defaults to 1024.
+The maximum number of T per group. Defaults to 1024.

 #### Defined in

-[index.ts:895](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L895)
+[index.ts:1235](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1235)

 ___

@@ -63,7 +63,7 @@ the number of cores on the machine.

 #### Defined in

-[index.ts:912](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L912)
+[index.ts:1252](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1252)

 ___

@@ -77,4 +77,4 @@ Defaults to 1024 * 1024.

 #### Defined in

-[index.ts:891](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L891)
+[index.ts:1231](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1231)
--- a/docs/src/javascript/interfaces/Connection.md
+++ b/docs/src/javascript/interfaces/Connection.md
@@ -22,6 +22,7 @@ Connection could be local against filesystem or remote against a server.
 - [dropTable](Connection.md#droptable)
 - [openTable](Connection.md#opentable)
 - [tableNames](Connection.md#tablenames)
+- [withMiddleware](Connection.md#withmiddleware)

 ## Properties

@@ -31,7 +32,7 @@ Connection could be local against filesystem or remote against a server.

 #### Defined in

-[index.ts:183](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L183)
+[index.ts:261](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L261)

 ## Methods

@@ -59,7 +60,7 @@ Creates a new Table, optionally initializing it with new data.

 #### Defined in

-[index.ts:207](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L207)
+[index.ts:285](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L285)

 ▸ **createTable**(`name`, `data`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>

@@ -70,7 +71,7 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |

 #### Returns

@@ -78,7 +79,7 @@ Creates a new Table and initialize it with new data.

 #### Defined in

-[index.ts:221](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L221)
+[index.ts:299](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L299)

 ▸ **createTable**(`name`, `data`, `options`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>

@@ -89,7 +90,7 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
 | `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |

 #### Returns
@@ -98,7 +99,7 @@ Creates a new Table and initialize it with new data.

 #### Defined in

-[index.ts:233](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L233)
+[index.ts:311](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L311)

 ▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`): `Promise`\<[`Table`](Table.md)\<`T`\>\>

@@ -115,7 +116,7 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
 | `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |

 #### Returns
@@ -124,7 +125,7 @@ Creates a new Table and initialize it with new data.

 #### Defined in

-[index.ts:246](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L246)
+[index.ts:324](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L324)

 ▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`, `options`): `Promise`\<[`Table`](Table.md)\<`T`\>\>

@@ -141,7 +142,7 @@ Creates a new Table and initialize it with new data.
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `name` | `string` | The name of the table. |
-| `data` | `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
 | `embeddings` | [`EmbeddingFunction`](EmbeddingFunction.md)\<`T`\> | An embedding function to use on this table |
 | `options` | [`WriteOptions`](WriteOptions.md) | The write options to use when creating the table. |

@@ -151,7 +152,7 @@ Creates a new Table and initialize it with new data.

 #### Defined in

-[index.ts:259](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L259)
+[index.ts:337](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L337)

 ___

@@ -173,7 +174,7 @@ Drop an existing table.

 #### Defined in

-[index.ts:270](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L270)
+[index.ts:348](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L348)

 ___

@@ -202,7 +203,7 @@ Open a table in the database.

 #### Defined in

-[index.ts:193](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L193)
+[index.ts:271](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L271)

 ___

@@ -216,4 +217,32 @@ ___

 #### Defined in

-[index.ts:185](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L185)
+[index.ts:263](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L263)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Connection`](Connection.md)
+
+Instrument the behavior of this Connection with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote Connections.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Connection`](Connection.md)
+
+- this Connection instrumented by the passed middleware
+
+#### Defined in
+
+[index.ts:360](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L360)
--- a/docs/src/javascript/interfaces/ConnectionOptions.md
+++ b/docs/src/javascript/interfaces/ConnectionOptions.md
@@ -10,7 +10,10 @@
 - [awsCredentials](ConnectionOptions.md#awscredentials)
 - [awsRegion](ConnectionOptions.md#awsregion)
 - [hostOverride](ConnectionOptions.md#hostoverride)
+- [readConsistencyInterval](ConnectionOptions.md#readconsistencyinterval)
 - [region](ConnectionOptions.md#region)
+- [storageOptions](ConnectionOptions.md#storageoptions)
+- [timeout](ConnectionOptions.md#timeout)
 - [uri](ConnectionOptions.md#uri)

 ## Properties
@@ -19,9 +22,13 @@

 • `Optional` **apiKey**: `string`

+API key for the remote connections
+
+Can also be passed by setting environment variable `LANCEDB_API_KEY`
+
 #### Defined in

-[index.ts:81](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L81)
+[index.ts:112](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L112)

 ___

@@ -33,9 +40,14 @@ User provided AWS crednetials.

 If not provided, LanceDB will use the default credentials provider chain.

+**`Deprecated`**
+
+Pass `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token`
+through `storageOptions` instead.
+
 #### Defined in

-[index.ts:75](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L75)
+[index.ts:92](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L92)

 ___

@@ -43,11 +55,15 @@ ___

 • `Optional` **awsRegion**: `string`

-AWS region to connect to. Default is defaultAwsRegion.
+AWS region to connect to. Default is defaultAwsRegion
+
+**`Deprecated`**
+
+Pass `region` through `storageOptions` instead.

 #### Defined in

-[index.ts:78](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L78)
+[index.ts:98](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L98)

 ___

@@ -55,13 +71,33 @@ ___

 • `Optional` **hostOverride**: `string`

-Override the host URL for the remote connections.
+Override the host URL for the remote connection.

 This is useful for local testing.

 #### Defined in

-[index.ts:91](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L91)
+[index.ts:122](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L122)
+
+___
+
+### readConsistencyInterval
+
+• `Optional` **readConsistencyInterval**: `number`
+
+(For LanceDB OSS only): The interval, in seconds, at which to check for
+updates to the table from other processes. If None, then consistency is not
+checked. For performance reasons, this is the default. For strong
+consistency, set this to zero seconds. Then every read will check for
+updates from other processes. As a compromise, you can set this to a
+non-zero value for eventual consistency. If more than that interval
+has passed since the last check, then the table will be checked for updates.
+Note: this consistency only applies to read operations. Write operations are
+always consistent.
+
+#### Defined in
+
+[index.ts:140](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L140)

 ___

@@ -69,11 +105,37 @@ ___

 • `Optional` **region**: `string`

-Region to connect
+Region to connect. Default is 'us-east-1'

 #### Defined in

-[index.ts:84](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L84)
+[index.ts:115](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L115)
+
+___
+
+### storageOptions
+
+• `Optional` **storageOptions**: `Record`\<`string`, `string`\>
+
+User provided options for object storage. For example, S3 credentials or request timeouts.
+
+The various options are described at https://lancedb.github.io/lancedb/guides/storage/
+
+#### Defined in
+
+[index.ts:105](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L105)
+
+___
+
+### timeout
+
+• `Optional` **timeout**: `number`
+
+Duration in milliseconds for request timeout. Default = 10,000 (10 seconds)
+
+#### Defined in
+
+[index.ts:127](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L127)

 ___

@@ -85,8 +147,8 @@ LanceDB database URI.

 - `/path/to/database` - local database
 - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- `db://host:port` - remote database (SaaS)
+- `db://host:port` - remote database (LanceDB cloud)

 #### Defined in

-[index.ts:69](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L69)
+[index.ts:83](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L83)
--- a/docs/src/javascript/interfaces/CreateTableOptions.md
+++ b/docs/src/javascript/interfaces/CreateTableOptions.md
@@ -26,7 +26,7 @@

 #### Defined in

-[index.ts:116](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L116)
+[index.ts:163](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L163)

 ___

@@ -36,7 +36,7 @@ ___

 #### Defined in

-[index.ts:122](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L122)
+[index.ts:169](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L169)

 ___

@@ -46,7 +46,7 @@ ___

 #### Defined in

-[index.ts:113](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L113)
+[index.ts:160](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L160)

 ___

@@ -56,7 +56,7 @@ ___

 #### Defined in

-[index.ts:119](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L119)
+[index.ts:166](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L166)

 ___

@@ -66,4 +66,4 @@ ___

 #### Defined in

-[index.ts:125](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L125)
+[index.ts:172](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L172)
--- a/docs/src/javascript/interfaces/EmbeddingFunction.md
+++ b/docs/src/javascript/interfaces/EmbeddingFunction.md
@@ -18,11 +18,29 @@ An embedding function that automatically creates vector representation for a giv

 ### Properties

+- [destColumn](EmbeddingFunction.md#destcolumn)
 - [embed](EmbeddingFunction.md#embed)
+- [embeddingDataType](EmbeddingFunction.md#embeddingdatatype)
+- [embeddingDimension](EmbeddingFunction.md#embeddingdimension)
+- [excludeSource](EmbeddingFunction.md#excludesource)
 - [sourceColumn](EmbeddingFunction.md#sourcecolumn)

 ## Properties

+### destColumn
+
+• `Optional` **destColumn**: `string`
+
+The name of the column that will contain the embedding
+
+By default this is "vector"
+
+#### Defined in
+
+[embedding/embedding_function.ts:49](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L49)
+
+___
+
 ### embed

 • **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
@@ -45,7 +63,54 @@ Creates a vector representation for the given values.

 #### Defined in

-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:62](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L62)
+
+___
+
+### embeddingDataType
+
+• `Optional` **embeddingDataType**: `Float`\<`Floats`\>
+
+The data type of the embedding
+
+The embedding function should return `number`.  This will be converted into
+an Arrow float array.  By default this will be Float32 but this property can
+be used to control the conversion.
+
+#### Defined in
+
+[embedding/embedding_function.ts:33](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L33)
+
+___
+
+### embeddingDimension
+
+• `Optional` **embeddingDimension**: `number`
+
+The dimension of the embedding
+
+This is optional, normally this can be determined by looking at the results of
+`embed`.  If this is not specified, and there is an attempt to apply the embedding
+to an empty table, then that process will fail.
+
+#### Defined in
+
+[embedding/embedding_function.ts:42](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L42)
+
+___
+
+### excludeSource
+
+• `Optional` **excludeSource**: `boolean`
+
+Should the source column be excluded from the resulting table
+
+By default the source column is included.  Set this to true and
+only the embedding will be stored.
+
+#### Defined in
+
+[embedding/embedding_function.ts:57](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L57)

 ___

@@ -57,4 +122,4 @@ The name of the column that will be used as input for the Embedding Function.

 #### Defined in

-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:24](https://github.com/lancedb/lancedb/blob/92179835/node/src/embedding/embedding_function.ts#L24)
--- a/docs/src/javascript/interfaces/IndexStats.md
+++ b/docs/src/javascript/interfaces/IndexStats.md
@@ -6,18 +6,51 @@

 ### Properties

+- [distanceType](IndexStats.md#distancetype)
+- [indexType](IndexStats.md#indextype)
 - [numIndexedRows](IndexStats.md#numindexedrows)
+- [numIndices](IndexStats.md#numindices)
 - [numUnindexedRows](IndexStats.md#numunindexedrows)

 ## Properties

+### distanceType
+
+• `Optional` **distanceType**: `string`
+
+#### Defined in
+
+[index.ts:728](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L728)
+
+___
+
+### indexType
+
+• **indexType**: `string`
+
+#### Defined in
+
+[index.ts:727](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L727)
+
+___
+
 ### numIndexedRows

 • **numIndexedRows**: ``null`` \| `number`

 #### Defined in

-[index.ts:478](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L478)
+[index.ts:725](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L725)
+
+___
+
+### numIndices
+
+• `Optional` **numIndices**: `number`
+
+#### Defined in
+
+[index.ts:729](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L729)

 ___

@@ -27,4 +60,4 @@ ___

 #### Defined in

-[index.ts:479](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L479)
+[index.ts:726](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L726)
--- a/docs/src/javascript/interfaces/IvfPQIndexConfig.md
+++ b/docs/src/javascript/interfaces/IvfPQIndexConfig.md
@@ -29,7 +29,7 @@ The column to be indexed

 #### Defined in

-[index.ts:942](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L942)
+[index.ts:1282](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1282)

 ___

@@ -41,7 +41,7 @@ Cache size of the index

 #### Defined in

-[index.ts:991](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L991)
+[index.ts:1331](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1331)

 ___

@@ -53,7 +53,7 @@ A unique name for the index

 #### Defined in

-[index.ts:947](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L947)
+[index.ts:1287](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1287)

 ___

@@ -65,7 +65,7 @@ The max number of iterations for kmeans training.

 #### Defined in

-[index.ts:962](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L962)
+[index.ts:1302](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1302)

 ___

@@ -77,7 +77,7 @@ Max number of iterations to train OPQ, if `use_opq` is true.

 #### Defined in

-[index.ts:981](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L981)
+[index.ts:1321](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1321)

 ___

@@ -89,7 +89,7 @@ Metric type, L2 or Cosine

 #### Defined in

-[index.ts:952](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L952)
+[index.ts:1292](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1292)

 ___

@@ -101,7 +101,7 @@ The number of bits to present one PQ centroid.

 #### Defined in

-[index.ts:976](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L976)
+[index.ts:1316](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1316)

 ___

@@ -113,7 +113,7 @@ The number of partitions this index

 #### Defined in

-[index.ts:957](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L957)
+[index.ts:1297](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1297)

 ___

@@ -125,7 +125,7 @@ Number of subvectors to build PQ code

 #### Defined in

-[index.ts:972](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L972)
+[index.ts:1312](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1312)

 ___

@@ -137,7 +137,7 @@ Replace an existing index with the same name if it exists.

 #### Defined in

-[index.ts:986](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L986)
+[index.ts:1326](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1326)

 ___

@@ -147,7 +147,7 @@ ___

 #### Defined in

-[index.ts:993](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L993)
+[index.ts:1333](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1333)

 ___

@@ -159,4 +159,4 @@ Train as optimized product quantization.

 #### Defined in

-[index.ts:967](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L967)
+[index.ts:1307](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1307)
--- a/docs/src/javascript/interfaces/MergeInsertArgs.md
+++ b/docs/src/javascript/interfaces/MergeInsertArgs.md
@@ -0,0 +1,73 @@
+[vectordb](../README.md) / [Exports](../modules.md) / MergeInsertArgs
+
+# Interface: MergeInsertArgs
+
+## Table of contents
+
+### Properties
+
+- [whenMatchedUpdateAll](MergeInsertArgs.md#whenmatchedupdateall)
+- [whenNotMatchedBySourceDelete](MergeInsertArgs.md#whennotmatchedbysourcedelete)
+- [whenNotMatchedInsertAll](MergeInsertArgs.md#whennotmatchedinsertall)
+
+## Properties
+
+### whenMatchedUpdateAll
+
+• `Optional` **whenMatchedUpdateAll**: `string` \| `boolean`
+
+If true then rows that exist in both the source table (new data) and
+the target table (old data) will be updated, replacing the old row
+with the corresponding matching row.
+
+If there are multiple matches then the behavior is undefined.
+Currently this causes multiple copies of the row to be created
+but that behavior is subject to change.
+
+Optionally, a filter can be specified.  This should be an SQL
+filter where fields with the prefix "target." refer to fields
+in the target table (old data) and fields with the prefix
+"source." refer to fields in the source table (new data).  For
+example, the filter "target.lastUpdated < source.lastUpdated" will
+only update matched rows when the incoming `lastUpdated` value is
+newer.
+
+Rows that do not match the filter will not be updated.  Rows that
+do not match the filter do become "not matched" rows.
+
+#### Defined in
+
+[index.ts:690](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L690)
+
+___
+
+### whenNotMatchedBySourceDelete
+
+• `Optional` **whenNotMatchedBySourceDelete**: `string` \| `boolean`
+
+If true then rows that exist only in the target table (old data)
+will be deleted.
+
+If this is a string then it will be treated as an SQL filter and
+only rows that both do not match any row in the source table and
+match the given filter will be deleted.
+
+This can be used to replace a selection of existing data with
+new data.
+
+#### Defined in
+
+[index.ts:707](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L707)
+
+___
+
+### whenNotMatchedInsertAll
+
+• `Optional` **whenNotMatchedInsertAll**: `boolean`
+
+If true then rows that exist only in the source table (new data)
+will be inserted into the target table.
+
+#### Defined in
+
+[index.ts:695](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L695)
--- a/docs/src/javascript/interfaces/Table.md
+++ b/docs/src/javascript/interfaces/Table.md
@@ -25,17 +25,26 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [delete](Table.md#delete)
 - [indexStats](Table.md#indexstats)
 - [listIndices](Table.md#listindices)
+- [mergeInsert](Table.md#mergeinsert)
 - [name](Table.md#name)
 - [overwrite](Table.md#overwrite)
 - [schema](Table.md#schema)
 - [search](Table.md#search)
 - [update](Table.md#update)

+### Methods
+
+- [addColumns](Table.md#addcolumns)
+- [alterColumns](Table.md#altercolumns)
+- [dropColumns](Table.md#dropcolumns)
+- [filter](Table.md#filter)
+- [withMiddleware](Table.md#withmiddleware)
+
 ## Properties

 ### add

-• **add**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
+• **add**: (`data`: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>

 #### Type declaration

@@ -47,7 +56,7 @@ Insert records into this Table.

 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |

 ##### Returns

@@ -57,27 +66,33 @@ The number of rows added to the table

 #### Defined in

-[index.ts:291](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L291)
+[index.ts:381](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L381)

 ___

 ### countRows

-• **countRows**: () => `Promise`\<`number`\>
+• **countRows**: (`filter?`: `string`) => `Promise`\<`number`\>

 #### Type declaration

-▸ (): `Promise`\<`number`\>
+▸ (`filter?`): `Promise`\<`number`\>

 Returns the number of rows in this table.

+##### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `filter?` | `string` |
+
 ##### Returns

 `Promise`\<`number`\>

 #### Defined in

-[index.ts:361](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L361)
+[index.ts:454](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L454)

 ___

@@ -107,17 +122,17 @@ VectorIndexParams.

 #### Defined in

-[index.ts:306](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L306)
+[index.ts:398](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L398)

 ___

 ### createScalarIndex

-• **createScalarIndex**: (`column`: `string`, `replace`: `boolean`) => `Promise`\<`void`\>
+• **createScalarIndex**: (`column`: `string`, `replace?`: `boolean`) => `Promise`\<`void`\>

 #### Type declaration

-▸ (`column`, `replace`): `Promise`\<`void`\>
+▸ (`column`, `replace?`): `Promise`\<`void`\>

 Create a scalar index on this Table for the given column

@@ -126,7 +141,7 @@ Create a scalar index on this Table for the given column
 | Name | Type | Description |
 | :------ | :------ | :------ |
 | `column` | `string` | The column to index |
-| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+| `replace?` | `boolean` | If false, fail if an index already exists on the column it is always set to true for remote connections Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |

 ##### Returns

@@ -142,7 +157,7 @@ await table.createScalarIndex('my_col')

 #### Defined in

-[index.ts:356](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L356)
+[index.ts:449](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L449)

 ___

@@ -194,17 +209,17 @@ await tbl.countRows() // Returns 1

 #### Defined in

-[index.ts:395](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L395)
+[index.ts:488](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L488)

 ___

 ### indexStats

-• **indexStats**: (`indexUuid`: `string`) => `Promise`\<[`IndexStats`](IndexStats.md)\>
+• **indexStats**: (`indexName`: `string`) => `Promise`\<[`IndexStats`](IndexStats.md)\>

 #### Type declaration

-▸ (`indexUuid`): `Promise`\<[`IndexStats`](IndexStats.md)\>
+▸ (`indexName`): `Promise`\<[`IndexStats`](IndexStats.md)\>

 Get statistics about an index.

@@ -212,7 +227,7 @@ Get statistics about an index.

 | Name | Type |
 | :------ | :------ |
-| `indexUuid` | `string` |
+| `indexName` | `string` |

 ##### Returns

@@ -220,7 +235,7 @@ Get statistics about an index.

 #### Defined in

-[index.ts:438](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L438)
+[index.ts:567](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L567)

 ___

@@ -240,7 +255,57 @@ List the indicies on this table.

 #### Defined in

-[index.ts:433](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L433)
+[index.ts:562](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L562)
+
+___
+
+### mergeInsert
+
+• **mergeInsert**: (`on`: `string`, `data`: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[], `args`: [`MergeInsertArgs`](MergeInsertArgs.md)) => `Promise`\<`void`\>
+
+#### Type declaration
+
+▸ (`on`, `data`, `args`): `Promise`\<`void`\>
+
+Runs a "merge insert" operation on the table
+
+This operation can add rows, update rows, and remove rows all in a single
+transaction. It is a very generic tool that can be used to create
+behaviors like "insert if not exists", "update or insert (i.e. upsert)",
+or even replace a portion of existing data with new data (e.g. replace
+all data where month="january")
+
+The merge insert operation works by combining new data from a
+**source table** with existing data in a **target table** by using a
+join.  There are three categories of records.
+
+"Matched" records are records that exist in both the source table and
+the target table. "Not matched" records exist only in the source table
+(e.g. these are new data) "Not matched by source" records exist only
+in the target table (this is old data)
+
+The MergeInsertArgs can be used to customize what should happen for
+each category of data.
+
+Please note that the data may appear to be reordered as part of this
+operation.  This is because updated rows will be deleted from the
+dataset and then reinserted at the end with the new values.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `on` | `string` | a column to join on. This is how records from the source table and target table are matched. |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | the new data to insert |
+| `args` | [`MergeInsertArgs`](MergeInsertArgs.md) | parameters controlling how the operation should behave |
+
+##### Returns
+
+`Promise`\<`void`\>
+
+#### Defined in
+
+[index.ts:553](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L553)

 ___

@@ -250,13 +315,13 @@ ___

 #### Defined in

-[index.ts:277](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L277)
+[index.ts:367](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L367)

 ___

 ### overwrite

-• **overwrite**: (`data`: `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>
+• **overwrite**: (`data`: `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[]) => `Promise`\<`number`\>

 #### Type declaration

@@ -268,7 +333,7 @@ Insert records into this Table, replacing its contents.

 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |

 ##### Returns

@@ -278,7 +343,7 @@ The number of rows added to the table

 #### Defined in

-[index.ts:299](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L299)
+[index.ts:389](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L389)

 ___

@@ -288,7 +353,7 @@ ___

 #### Defined in

-[index.ts:440](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L440)
+[index.ts:571](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L571)

 ___

@@ -314,7 +379,7 @@ Creates a search query to find the nearest neighbors of the given search term

 #### Defined in

-[index.ts:283](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L283)
+[index.ts:373](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L373)

 ___

@@ -365,4 +430,123 @@ let results = await tbl.search([1, 1]).execute();

 #### Defined in

-[index.ts:428](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L428)
+[index.ts:521](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L521)
+
+## Methods
+
+### addColumns
+
+▸ **addColumns**(`newColumnTransforms`): `Promise`\<`void`\>
+
+Add new columns with defined values.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `newColumnTransforms` | \{ `name`: `string` ; `valueSql`: `string`  }[] | pairs of column names and the SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns in the table. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Defined in
+
+[index.ts:582](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L582)
+
+___
+
+### alterColumns
+
+▸ **alterColumns**(`columnAlterations`): `Promise`\<`void`\>
+
+Alter the name or nullability of columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnAlterations` | [`ColumnAlteration`](ColumnAlteration.md)[] | One or more alterations to apply to columns. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Defined in
+
+[index.ts:591](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L591)
+
+___
+
+### dropColumns
+
+▸ **dropColumns**(`columnNames`): `Promise`\<`void`\>
+
+Drop one or more columns from the dataset
+
+This is a metadata-only operation and does not remove the data from the
+underlying storage. In order to remove the data, you must subsequently
+call ``compact_files`` to rewrite the data without the removed columns and
+then call ``cleanup_files`` to remove the old files.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `columnNames` | `string`[] | The names of the columns to drop. These can be nested column references (e.g. "a.b.c") or top-level column names (e.g. "a"). |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Defined in
+
+[index.ts:605](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L605)
+
+___
+
+### filter
+
+▸ **filter**(`value`): [`Query`](../classes/Query.md)\<`T`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `value` | `string` |
+
+#### Returns
+
+[`Query`](../classes/Query.md)\<`T`\>
+
+#### Defined in
+
+[index.ts:569](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L569)
+
+___
+
+### withMiddleware
+
+▸ **withMiddleware**(`middleware`): [`Table`](Table.md)\<`T`\>
+
+Instrument the behavior of this Table with middleware.
+
+The middleware will be called in the order they are added.
+
+Currently this functionality is only supported for remote tables.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `middleware` | `HttpMiddleware` |
+
+#### Returns
+
+[`Table`](Table.md)\<`T`\>
+
+- this Table instrumented by the passed middleware
+
+#### Defined in
+
+[index.ts:617](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L617)
--- a/docs/src/javascript/interfaces/UpdateArgs.md
+++ b/docs/src/javascript/interfaces/UpdateArgs.md
@@ -20,7 +20,7 @@ new values to set

 #### Defined in

-[index.ts:454](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L454)
+[index.ts:652](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L652)

 ___

@@ -33,4 +33,4 @@ in which case all rows will be updated.

 #### Defined in

-[index.ts:448](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L448)
+[index.ts:646](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L646)
--- a/docs/src/javascript/interfaces/UpdateSqlArgs.md
+++ b/docs/src/javascript/interfaces/UpdateSqlArgs.md
@@ -20,7 +20,7 @@ new values to set as SQL expressions.

 #### Defined in

-[index.ts:468](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L468)
+[index.ts:666](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L666)

 ___

@@ -33,4 +33,4 @@ in which case all rows will be updated.

 #### Defined in

-[index.ts:462](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L462)
+[index.ts:660](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L660)
--- a/docs/src/javascript/interfaces/VectorIndex.md
+++ b/docs/src/javascript/interfaces/VectorIndex.md
@@ -8,6 +8,7 @@

 - [columns](VectorIndex.md#columns)
 - [name](VectorIndex.md#name)
+- [status](VectorIndex.md#status)
 - [uuid](VectorIndex.md#uuid)

 ## Properties
@@ -18,7 +19,7 @@

 #### Defined in

-[index.ts:472](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L472)
+[index.ts:718](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L718)

 ___

@@ -28,7 +29,17 @@ ___

 #### Defined in

-[index.ts:473](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L473)
+[index.ts:719](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L719)
+
+___
+
+### status
+
+• **status**: [`IndexStatus`](../enums/IndexStatus.md)
+
+#### Defined in
+
+[index.ts:721](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L721)

 ___

@@ -38,4 +49,4 @@ ___

 #### Defined in

-[index.ts:474](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L474)
+[index.ts:720](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L720)
--- a/docs/src/javascript/interfaces/WriteOptions.md
+++ b/docs/src/javascript/interfaces/WriteOptions.md
@@ -24,4 +24,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation

 #### Defined in

-[index.ts:1015](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1015)
+[index.ts:1355](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1355)
--- a/docs/src/javascript/modules.md
+++ b/docs/src/javascript/modules.md
@@ -6,6 +6,7 @@

 ### Enumerations

+- [IndexStatus](enums/IndexStatus.md)
 - [MetricType](enums/MetricType.md)
 - [WriteMode](enums/WriteMode.md)

@@ -14,6 +15,7 @@
 - [DefaultWriteOptions](classes/DefaultWriteOptions.md)
 - [LocalConnection](classes/LocalConnection.md)
 - [LocalTable](classes/LocalTable.md)
+- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
 - [Query](classes/Query.md)

@@ -21,6 +23,7 @@

 - [AwsCredentials](interfaces/AwsCredentials.md)
 - [CleanupStats](interfaces/CleanupStats.md)
+- [ColumnAlteration](interfaces/ColumnAlteration.md)
 - [CompactionMetrics](interfaces/CompactionMetrics.md)
 - [CompactionOptions](interfaces/CompactionOptions.md)
 - [Connection](interfaces/Connection.md)
@@ -29,6 +32,7 @@
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
 - [IndexStats](interfaces/IndexStats.md)
 - [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
+- [MergeInsertArgs](interfaces/MergeInsertArgs.md)
 - [Table](interfaces/Table.md)
 - [UpdateArgs](interfaces/UpdateArgs.md)
 - [UpdateSqlArgs](interfaces/UpdateSqlArgs.md)
@@ -42,7 +46,9 @@
 ### Functions

 - [connect](modules.md#connect)
+- [convertToTable](modules.md#converttotable)
 - [isWriteOptions](modules.md#iswriteoptions)
+- [makeArrowTable](modules.md#makearrowtable)

 ## Type Aliases

@@ -52,7 +58,7 @@

 #### Defined in

-[index.ts:996](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L996)
+[index.ts:1336](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1336)

 ## Functions

@@ -62,11 +68,11 @@

 Connect to a LanceDB instance at the given URI.

-Accpeted formats:
+Accepted formats:

 - `/path/to/database` - local database
 - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- `db://host:port` - remote database (SaaS)
+- `db://host:port` - remote database (LanceDB cloud)

 #### Parameters

@@ -84,7 +90,7 @@ Accpeted formats:

 #### Defined in

-[index.ts:141](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L141)
+[index.ts:188](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L188)

 ▸ **connect**(`opts`): `Promise`\<[`Connection`](interfaces/Connection.md)\>

@@ -102,7 +108,35 @@ Connect to a LanceDB instance with connection options.

 #### Defined in

-[index.ts:147](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L147)
+[index.ts:194](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L194)
+
+___
+
+### convertToTable
+
+▸ **convertToTable**\<`T`\>(`data`, `embeddings?`, `makeTableOptions?`): `Promise`\<`ArrowTable`\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `data` | `Record`\<`string`, `unknown`\>[] |
+| `embeddings?` | [`EmbeddingFunction`](interfaces/EmbeddingFunction.md)\<`T`\> |
+| `makeTableOptions?` | `Partial`\<[`MakeArrowTableOptions`](classes/MakeArrowTableOptions.md)\> |
+
+#### Returns
+
+`Promise`\<`ArrowTable`\>
+
+#### Defined in
+
+[arrow.ts:465](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L465)

 ___

@@ -122,4 +156,116 @@ value is WriteOptions

 #### Defined in

-[index.ts:1022](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1022)
+[index.ts:1362](https://github.com/lancedb/lancedb/blob/92179835/node/src/index.ts#L1362)
+
+___
+
+### makeArrowTable
+
+▸ **makeArrowTable**(`data`, `options?`): `ArrowTable`
+
+An enhanced version of the makeTable function from Apache Arrow
+that supports nested fields and embeddings columns.
+
+This function converts an array of Record<String, any> (row-major JS objects)
+to an Arrow Table (a columnar structure)
+
+Note that it currently does not support nulls.
+
+If a schema is provided then it will be used to determine the resulting array
+types.  Fields will also be reordered to fit the order defined by the schema.
+
+If a schema is not provided then the types will be inferred and the field order
+will be controlled by the order of properties in the first record.
+
+If the input is empty then a schema must be provided to create an empty table.
+
+When a schema is not specified then data types will be inferred.  The inference
+rules are as follows:
+
+ - boolean => Bool
+ - number => Float64
+ - String => Utf8
+ - Buffer => Binary
+ - Record<String, any> => Struct
+ - Array<any> => List
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`\<`string`, `any`\>[] | input data |
+| `options?` | `Partial`\<[`MakeArrowTableOptions`](classes/MakeArrowTableOptions.md)\> | options to control the makeArrowTable call. |
+
+#### Returns
+
+`ArrowTable`
+
+**`Example`**
+
+```ts
+
+import { fromTableToBuffer, makeArrowTable } from "../arrow";
+import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+
+const schema = new Schema([
+  new Field("a", new Int32()),
+  new Field("b", new Float32()),
+  new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ ]);
+ const table = makeArrowTable([
+   { a: 1, b: 2, c: [1, 2, 3] },
+   { a: 4, b: 5, c: [4, 5, 6] },
+   { a: 7, b: 8, c: [7, 8, 9] },
+ ], { schema });
+```
+
+By default it assumes that the column named `vector` is a vector column
+and it will be converted into a fixed size list array of type float32.
+The `vectorColumns` option can be used to support other vector column
+names and data types.
+
+```ts
+
+const schema = new Schema([
+   new Field("a", new Float64()),
+   new Field("b", new Float64()),
+   new Field(
+     "vector",
+     new FixedSizeList(3, new Field("item", new Float32()))
+   ),
+ ]);
+ const table = makeArrowTable([
+   { a: 1, b: 2, vector: [1, 2, 3] },
+   { a: 4, b: 5, vector: [4, 5, 6] },
+   { a: 7, b: 8, vector: [7, 8, 9] },
+ ]);
+ assert.deepEqual(table.schema, schema);
+```
+
+You can specify the vector column types and names using the options as well
+
+```typescript
+
+const schema = new Schema([
+   new Field('a', new Float64()),
+   new Field('b', new Float64()),
+   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ ]);
+const table = makeArrowTable([
+   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ ], {
+   vectorColumns: {
+     vec1: { type: new Float16() },
+     vec2: { type: new Float16() }
+   }
+ }
+assert.deepEqual(table.schema, schema)
+```
+
+#### Defined in
+
+[arrow.ts:198](https://github.com/lancedb/lancedb/blob/92179835/node/src/arrow.ts#L198)
--- a/docs/src/js/interfaces/FtsOptions.md
+++ b/docs/src/js/interfaces/FtsOptions.md
@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / FtsOptions
+
+# Interface: FtsOptions
+
+Options to create an `FTS` index
+
+## Properties
+
+### withPosition?
+
+> `optional` **withPosition**: `boolean`
+
+Whether to store the positions of the term in the document.
+
+If this is true then the index will store the positions of the term in the document.
+This allows phrase queries to be run. But it also increases the size of the index,
+and the time to build the index.
+
+The default value is true.
+
+***
--- a/docs/src/styles/extra.css
+++ b/docs/src/styles/extra.css
@@ -39,4 +39,46 @@
      height: 1.2rem;
      margin-top: -.1rem;
    }
-  }
+  }
+
+/* remove pilcrow as permanent link and add chain icon similar to github https://github.com/squidfunk/mkdocs-material/discussions/3535 */
+
+.headerlink {
+	--permalink-size: 16px; /* for font-relative sizes, 0.6em is a good choice */
+	--permalink-spacing: 4px;
+
+	width: calc(var(--permalink-size) + var(--permalink-spacing));
+	height: var(--permalink-size);
+	vertical-align: middle;
+	background-color: var(--md-default-fg-color--lighter);
+	background-size: var(--permalink-size);
+	mask-size: var(--permalink-size);
+	-webkit-mask-size: var(--permalink-size);
+	mask-repeat: no-repeat;
+	-webkit-mask-repeat: no-repeat;
+	visibility: visible;
+	mask-image: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="16" height="16" aria-hidden="true"><path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"></path></svg>');
+	-webkit-mask-image: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="16" height="16" aria-hidden="true"><path fill-rule="evenodd" d="M7.775 3.275a.75.75 0 001.06 1.06l1.25-1.25a2 2 0 112.83 2.83l-2.5 2.5a2 2 0 01-2.83 0 .75.75 0 00-1.06 1.06 3.5 3.5 0 004.95 0l2.5-2.5a3.5 3.5 0 00-4.95-4.95l-1.25 1.25zm-4.69 9.64a2 2 0 010-2.83l2.5-2.5a2 2 0 012.83 0 .75.75 0 001.06-1.06 3.5 3.5 0 00-4.95 0l-2.5 2.5a3.5 3.5 0 004.95 4.95l1.25-1.25a.75.75 0 00-1.06-1.06l-1.25 1.25a2 2 0 01-2.83 0z"></path></svg>');
+}
+
+[id]:target .headerlink {
+	background-color: var(--md-typeset-a-color);
+}
+
+.headerlink:hover {
+	background-color: var(--md-accent-fg-color) !important;
+}
+
+@media screen and (min-width: 76.25em) {
+	h1, h2, h3, h4, h5, h6 {
+		display: flex;
+		align-items: center;
+		flex-direction: row;
+		column-gap: 0.2em; /* fixes spaces in titles */
+	}
+
+	.headerlink {
+		order: -1;
+		margin-left: calc(var(--permalink-size) * -1 - var(--permalink-spacing)) !important;
+	}
+}
--- a/docs/src/troubleshooting.md
+++ b/docs/src/troubleshooting.md
@@ -0,0 +1,33 @@
+## Getting help
+
+The following sections provide various diagnostics and troubleshooting tips for LanceDB.
+These can help you provide additional information when asking questions or making
+error reports.
+
+For trouble shooting, the best place to ask is in our Discord, under the relevant
+language channel. By asking in the language-specific channel, it makes it more
+likely that someone who knows the answer will see your question.
+
+## Enabling logging
+
+To provide more information, especially for LanceDB Cloud related issues, enable
+debug logging. You can set the `LANCEDB_LOG` environment variable:
+
+```shell
+export LANCEDB_LOG=debug
+```
+
+You can turn off colors and formatting in the logs by setting
+
+```shell
+export LANCEDB_LOG_STYLE=never
+```
+
+## Explaining query plans
+
+If you have slow queries or unexpected query results, it can be helpful to
+print the resolved query plan. You can use the `explain_plan` method to do this:
+
+* Python Sync: [LanceQueryBuilder.explain_plan][lancedb.query.LanceQueryBuilder.explain_plan]
+* Python Async: [AsyncQueryBase.explain_plan][lancedb.query.AsyncQueryBase.explain_plan]
+* Node @lancedb/lancedb: [LanceQueryBuilder.explainPlan](/lancedb/js/classes/QueryBase/#explainplan)
--- a/docs/test/requirements.txt
+++ b/docs/test/requirements.txt
@@ -3,7 +3,7 @@ numpy
 pandas
 pylance
 duckdb
+tantivy==0.20.1
 --extra-index-url https://download.pytorch.org/whl/cpu
 torch
 polars>=0.19, <=1.3.0
-
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -8,7 +8,7 @@
    <parent>
        <groupId>com.lancedb</groupId>
        <artifactId>lancedb-parent</artifactId>
-        <version>0.11.0-beta.1</version>
+        <version>0.13.0-beta.1</version>
        <relativePath>../pom.xml</relativePath>
    </parent>

--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.11.0-beta.1</version>
+    <version>0.13.0-beta.1</version>
    <packaging>pom</packaging>

    <name>LanceDB Parent</name>
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.11.0-beta.1",
+  "version": "0.13.0-beta.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.11.0-beta.1",
+      "version": "0.13.0-beta.0",
      "cpu": [
        "x64",
        "arm64"
@@ -52,11 +52,11 @@
        "uuid": "^9.0.0"
      },
      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.11.0-beta.1",
-        "@lancedb/vectordb-darwin-x64": "0.11.0-beta.1",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.11.0-beta.1",
-        "@lancedb/vectordb-linux-x64-gnu": "0.11.0-beta.1",
-        "@lancedb/vectordb-win32-x64-msvc": "0.11.0-beta.1"
+        "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.13.0-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.0"
      },
      "peerDependencies": {
        "@apache-arrow/ts": "^14.0.2",
@@ -326,66 +326,6 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.11.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.11.0-beta.1.tgz",
-      "integrity": "sha512-qKQbFJwstMQEO2MVkkipyDxmH3/KafkuC4xfU8LjMtZ98ZGTQIW47t/OyftiUXYWcjsVxeXI3l2m9MCozFOdhg==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.11.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.11.0-beta.1.tgz",
-      "integrity": "sha512-245Q5hjQKljczBcDLbiq3N5fmUaY2zFRHoW6SBxOziQwyMphhLDSTNkAYkc3JnrQvf6dMolVYWigOsRVCFj56A==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.11.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.11.0-beta.1.tgz",
-      "integrity": "sha512-B4z6sx4X6uqGDnQm3zL5mL47Agn4X4spf/nlxtrUWEfiOAyp9Iw465UQMmrbnodi+4k/BNjCNZNMFSjMOSsrcA==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.11.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.11.0-beta.1.tgz",
-      "integrity": "sha512-0vWcPqpe3to78bYkc+3XWZToRu6TMrhLJAxC9cnV5d9GMuN1VbDoLqD8QPRWkoEr9Nk7jdIRKEBUwfq5yGOFLw==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.11.0-beta.1",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.11.0-beta.1.tgz",
-      "integrity": "sha512-jU/+w2TfA4HKOZkib1UP4kIpaLgu+88S/t+Ccde67w/4qQuP0uAixTAls1WE4mtlf6pOnG0A1ILTY98nVkIQ3A==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
    "node_modules/@neon-rs/cli": {
      "version": "0.0.160",
      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
--- a/node/package.json
+++ b/node/package.json
@@ -1,6 +1,6 @@
 {
  "name": "vectordb",
-  "version": "0.11.0-beta.1",
+  "version": "0.13.0-beta.1",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
@@ -88,10 +88,10 @@
    }
  },
  "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.11.0-beta.1",
-    "@lancedb/vectordb-darwin-x64": "0.11.0-beta.1",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.11.0-beta.1",
-    "@lancedb/vectordb-linux-x64-gnu": "0.11.0-beta.1",
-    "@lancedb/vectordb-win32-x64-msvc": "0.11.0-beta.1"
+    "@lancedb/vectordb-darwin-arm64": "0.13.0-beta.1",
+    "@lancedb/vectordb-darwin-x64": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.13.0-beta.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.13.0-beta.1"
  }
 }
--- a/node/src/index.ts
+++ b/node/src/index.ts
@@ -564,7 +564,7 @@ export interface Table<T = number[]> {
  /**
   * Get statistics about an index.
   */
-  indexStats: (indexUuid: string) => Promise<IndexStats>
+  indexStats: (indexName: string) => Promise<IndexStats>

  filter(value: string): Query<T>

@@ -1164,8 +1164,8 @@ export class LocalTable<T = number[]> implements Table<T> {
    return tableListIndices.call(this._tbl);
  }

-  async indexStats(indexUuid: string): Promise<IndexStats> {
-    return tableIndexStats.call(this._tbl, indexUuid);
+  async indexStats(indexName: string): Promise<IndexStats> {
+    return tableIndexStats.call(this._tbl, indexName);
  }

  get schema(): Promise<Schema> {
--- a/node/src/remote/client.ts
+++ b/node/src/remote/client.ts
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-import axios, { type AxiosResponse, type ResponseType } from 'axios'
+import axios, { type AxiosError, type AxiosResponse, type ResponseType } from 'axios'

 import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'

@@ -197,7 +197,7 @@ export class HttpLancedbClient {
      response = await callWithMiddlewares(req, this._middlewares)
      return response
    } catch (err: any) {
-      console.error('error: ', err)
+      console.error(serializeErrorAsJson(err))
      if (err.response === undefined) {
        throw new Error(`Network Error: ${err.message as string}`)
      }
@@ -247,7 +247,8 @@ export class HttpLancedbClient {

      // return response
    } catch (err: any) {
-      console.error('error: ', err)
+      console.error(serializeErrorAsJson(err))
+
      if (err.response === undefined) {
        throw new Error(`Network Error: ${err.message as string}`)
      }
@@ -287,3 +288,15 @@ export class HttpLancedbClient {
    return clone
  }
 }
+
+function serializeErrorAsJson(err: AxiosError) {
+  const error = JSON.parse(JSON.stringify(err, Object.getOwnPropertyNames(err)))
+  error.response = err.response != null
+      ? JSON.parse(JSON.stringify(
+        err.response,
+        // config contains the request data, too noisy
+        Object.getOwnPropertyNames(err.response).filter(prop => prop !== 'config')
+      ))
+      : null
+  return JSON.stringify({ error })
+}
--- a/node/src/remote/index.ts
+++ b/node/src/remote/index.ts
@@ -517,9 +517,9 @@ export class RemoteTable<T = number[]> implements Table<T> {
    }))
  }

-  async indexStats (indexUuid: string): Promise<IndexStats> {
+  async indexStats (indexName: string): Promise<IndexStats> {
    const results = await this._client.post(
-      `/v1/table/${encodeURIComponent(this._name)}/index/${indexUuid}/stats/`
+      `/v1/table/${encodeURIComponent(this._name)}/index/${indexName}/stats/`
    )
    const body = await results.body()
    return {
--- a/nodejs/Cargo.toml
+++ b/nodejs/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.11.0-beta.1"
+version = "0.13.0-beta.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -13,6 +13,7 @@ crate-type = ["cdylib"]

 [dependencies]
 arrow-ipc.workspace = true
+env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", features = ["remote"] }
 napi = { version = "2.16.8", default-features = false, features = [
@@ -22,6 +23,7 @@ napi = { version = "2.16.8", default-features = false, features = [
 napi-derive = "2.16.4"
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }
+log.workspace = true

 [build-dependencies]
 napi-build = "2.1"
--- a/nodejs/test/remote.test.ts
+++ b/nodejs/test/remote.test.ts
@@ -90,4 +90,29 @@ describe("remote connection", () => {
      },
    );
  });
+
+  it("shows the full error messages on retry errors", async () => {
+    await withMockDatabase(
+      (_req, res) => {
+        // We retry on 500 errors, so we return 500s until the client gives up.
+        res.writeHead(500).end("Internal Server Error");
+      },
+      async (db) => {
+        try {
+          await db.tableNames();
+          fail("expected an error");
+          // biome-ignore lint/suspicious/noExplicitAny: skip
+        } catch (e: any) {
+          expect(e.message).toContain("Hit retry limit for request_id=");
+          expect(e.message).toContain("Caused by: Http error");
+          expect(e.message).toContain("500 Internal Server Error");
+        }
+      },
+      {
+        clientConfig: {
+          retryConfig: { retries: 2 },
+        },
+      },
+    );
+  });
 });
--- a/nodejs/test/table.test.ts
+++ b/nodejs/test/table.test.ts
@@ -402,6 +402,40 @@ describe("When creating an index", () => {
    expect(rst.numRows).toBe(1);
  });

+  it("should be able to query unindexed data", async () => {
+    await tbl.createIndex("vec");
+    await tbl.add([
+      {
+        id: 300,
+        vec: Array(32)
+          .fill(1)
+          .map(() => Math.random()),
+        tags: [],
+      },
+    ]);
+
+    const plan1 = await tbl.query().nearestTo(queryVec).explainPlan(true);
+    expect(plan1).toMatch("LanceScan");
+
+    const plan2 = await tbl
+      .query()
+      .nearestTo(queryVec)
+      .fastSearch()
+      .explainPlan(true);
+    expect(plan2).not.toMatch("LanceScan");
+  });
+
+  it("should be able to query with row id", async () => {
+    const results = await tbl
+      .query()
+      .nearestTo(queryVec)
+      .withRowId()
+      .limit(1)
+      .toArray();
+    expect(results.length).toBe(1);
+    expect(results[0]).toHaveProperty("_rowid");
+  });
+
  it("should allow parameters to be specified", async () => {
    await tbl.createIndex("vec", {
      config: Index.ivfPq({
--- a/nodejs/lancedb/query.ts
+++ b/nodejs/lancedb/query.ts
@@ -239,6 +239,29 @@ export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
    return this;
  }

+  /**
+   * Skip searching un-indexed data. This can make search faster, but will miss
+   * any data that is not yet indexed.
+   *
+   * Use {@link lancedb.Table#optimize} to index all un-indexed data.
+   */
+  fastSearch(): this {
+    this.doCall((inner: NativeQueryType) => inner.fastSearch());
+    return this;
+  }
+
+  /**
+   * Whether to return the row id in the results.
+   *
+   * This column can be used to match results between different queries. For
+   * example, to match results from a full text search and a vector search in
+   * order to perform hybrid search.
+   */
+  withRowId(): this {
+    this.doCall((inner: NativeQueryType) => inner.withRowId());
+    return this;
+  }
+
  protected nativeExecute(
    options?: Partial<QueryExecutionOptions>,
  ): Promise<NativeBatchIterator> {
--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.11.0-beta.1",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
--- a/nodejs/npm/darwin-x64/package.json
+++ b/nodejs/npm/darwin-x64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.11.0-beta.1",
+	"version": "0.13.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.11.0-beta.1",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.11.0-beta.1",
+	"version": "0.13.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.11.0-beta.1",
+	"version": "0.13.0-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.11.0-beta.1",
+  "version": "0.12.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.11.0-beta.1",
+      "version": "0.12.0",
      "cpu": [
        "x64",
        "arm64"
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -10,7 +10,7 @@
    "vector database",
    "ann"
  ],
-  "version": "0.11.0-beta.1",
+  "version": "0.13.0-beta.1",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
--- a/nodejs/src/connection.rs
+++ b/nodejs/src/connection.rs
@@ -18,6 +18,7 @@ use std::str::FromStr;
 use napi::bindgen_prelude::*;
 use napi_derive::*;

+use crate::error::{convert_error, NapiErrorExt};
 use crate::table::Table;
 use crate::ConnectionOptions;
 use lancedb::connection::{
@@ -86,12 +87,7 @@ impl Connection {
            builder = builder.host_override(&host_override);
        }

-        Ok(Self::inner_new(
-            builder
-                .execute()
-                .await
-                .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
-        ))
+        Ok(Self::inner_new(builder.execute().await.default_error()?))
    }

    #[napi]
@@ -123,9 +119,7 @@ impl Connection {
        if let Some(limit) = limit {
            op = op.limit(limit);
        }
-        op.execute()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("{}", e)))
+        op.execute().await.default_error()
    }

    /// Create table from a Apache Arrow IPC (file) buffer.
@@ -156,17 +150,13 @@ impl Connection {
        }
        if let Some(data_storage_option) = data_storage_options.as_ref() {
            builder = builder.data_storage_version(
-                LanceFileVersion::from_str(data_storage_option)
-                    .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
+                LanceFileVersion::from_str(data_storage_option).map_err(|e| convert_error(&e))?,
            );
        }
        if let Some(enable_v2_manifest_paths) = enable_v2_manifest_paths {
            builder = builder.enable_v2_manifest_paths(enable_v2_manifest_paths);
        }
-        let tbl = builder
-            .execute()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        let tbl = builder.execute().await.default_error()?;
        Ok(Table::new(tbl))
    }

@@ -195,17 +185,13 @@ impl Connection {
        }
        if let Some(data_storage_option) = data_storage_options.as_ref() {
            builder = builder.data_storage_version(
-                LanceFileVersion::from_str(data_storage_option)
-                    .map_err(|e| napi::Error::from_reason(format!("{}", e)))?,
+                LanceFileVersion::from_str(data_storage_option).map_err(|e| convert_error(&e))?,
            );
        }
        if let Some(enable_v2_manifest_paths) = enable_v2_manifest_paths {
            builder = builder.enable_v2_manifest_paths(enable_v2_manifest_paths);
        }
-        let tbl = builder
-            .execute()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        let tbl = builder.execute().await.default_error()?;
        Ok(Table::new(tbl))
    }

@@ -225,19 +211,13 @@ impl Connection {
        if let Some(index_cache_size) = index_cache_size {
            builder = builder.index_cache_size(index_cache_size);
        }
-        let tbl = builder
-            .execute()
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("{}", e)))?;
+        let tbl = builder.execute().await.default_error()?;
        Ok(Table::new(tbl))
    }

    /// Drop table with the name. Or raise an error if the table does not exist.
    #[napi(catch_unwind)]
    pub async fn drop_table(&self, name: String) -> napi::Result<()> {
-        self.get_inner()?
-            .drop_table(&name)
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("{}", e)))
+        self.get_inner()?.drop_table(&name).await.default_error()
    }
 }
--- a/nodejs/src/error.rs
+++ b/nodejs/src/error.rs
@@ -7,6 +7,31 @@ pub trait NapiErrorExt<T> {

 impl<T> NapiErrorExt<T> for std::result::Result<T, lancedb::Error> {
    fn default_error(self) -> Result<T> {
-        self.map_err(|err| napi::Error::from_reason(err.to_string()))
+        self.map_err(|err| convert_error(&err))
    }
 }
+
+pub fn convert_error(err: &dyn std::error::Error) -> napi::Error {
+    let mut message = err.to_string();
+
+    // Append causes
+    let mut cause = err.source();
+    let mut indent = 2;
+    while let Some(err) = cause {
+        let cause_message = format!("Caused by: {}", err);
+        message.push_str(&indent_string(&cause_message, indent));
+
+        cause = err.source();
+        indent += 2;
+    }
+
+    napi::Error::from_reason(message)
+}
+
+fn indent_string(s: &str, amount: usize) -> String {
+    let indent = " ".repeat(amount);
+    s.lines()
+        .map(|line| format!("{}{}", indent, line))
+        .collect::<Vec<_>>()
+        .join("\n")
+}
--- a/nodejs/src/lib.rs
+++ b/nodejs/src/lib.rs
@@ -14,6 +14,7 @@

 use std::collections::HashMap;

+use env_logger::Env;
 use napi_derive::*;

 mod connection;
@@ -77,3 +78,11 @@ pub struct WriteOptions {
 pub struct OpenTableOptions {
    pub storage_options: Option<HashMap<String, String>>,
 }
+
+#[napi::module_init]
+fn init() {
+    let env = Env::new()
+        .filter_or("LANCEDB_LOG", "warn")
+        .write_style("LANCEDB_LOG_STYLE");
+    env_logger::init_from_env(env);
+}
--- a/nodejs/src/merge.rs
+++ b/nodejs/src/merge.rs
@@ -2,6 +2,8 @@ use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeIns
 use napi::bindgen_prelude::*;
 use napi_derive::napi;

+use crate::error::convert_error;
+
 #[napi]
 #[derive(Clone)]
 /// A builder used to create and run a merge insert operation
@@ -35,14 +37,18 @@ impl NativeMergeInsertBuilder {
    pub async fn execute(&self, buf: Buffer) -> napi::Result<()> {
        let data = ipc_file_to_batches(buf.to_vec())
            .and_then(IntoArrow::into_arrow)
-            .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+            .map_err(|e| {
+                napi::Error::from_reason(format!("Failed to read IPC file: {}", convert_error(&e)))
+            })?;

        let this = self.clone();

-        this.inner
-            .execute(data)
-            .await
-            .map_err(|e| napi::Error::from_reason(format!("Failed to execute merge insert: {}", e)))
+        this.inner.execute(data).await.map_err(|e| {
+            napi::Error::from_reason(format!(
+                "Failed to execute merge insert: {}",
+                convert_error(&e)
+            ))
+        })
    }
 }

--- a/nodejs/src/query.rs
+++ b/nodejs/src/query.rs
@@ -22,6 +22,7 @@ use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;

+use crate::error::convert_error;
 use crate::error::NapiErrorExt;
 use crate::iterator::RecordBatchIterator;
 use crate::util::parse_distance_type;
@@ -79,6 +80,16 @@ impl Query {
        Ok(VectorQuery { inner })
    }

+    #[napi]
+    pub fn fast_search(&mut self) {
+        self.inner = self.inner.clone().fast_search();
+    }
+
+    #[napi]
+    pub fn with_row_id(&mut self) {
+        self.inner = self.inner.clone().with_row_id();
+    }
+
    #[napi(catch_unwind)]
    pub async fn execute(
        &self,
@@ -93,7 +104,10 @@ impl Query {
            .execute_with_options(execution_opts)
            .await
            .map_err(|e| {
-                napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+                napi::Error::from_reason(format!(
+                    "Failed to execute query stream: {}",
+                    convert_error(&e)
+                ))
            })?;
        Ok(RecordBatchIterator::new(inner_stream))
    }
@@ -101,7 +115,10 @@ impl Query {
    #[napi]
    pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
        self.inner.explain_plan(verbose).await.map_err(|e| {
-            napi::Error::from_reason(format!("Failed to retrieve the query plan: {}", e))
+            napi::Error::from_reason(format!(
+                "Failed to retrieve the query plan: {}",
+                convert_error(&e)
+            ))
        })
    }
 }
@@ -176,6 +193,16 @@ impl VectorQuery {
        self.inner = self.inner.clone().offset(offset as usize);
    }

+    #[napi]
+    pub fn fast_search(&mut self) {
+        self.inner = self.inner.clone().fast_search();
+    }
+
+    #[napi]
+    pub fn with_row_id(&mut self) {
+        self.inner = self.inner.clone().with_row_id();
+    }
+
    #[napi(catch_unwind)]
    pub async fn execute(
        &self,
@@ -190,7 +217,10 @@ impl VectorQuery {
            .execute_with_options(execution_opts)
            .await
            .map_err(|e| {
-                napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+                napi::Error::from_reason(format!(
+                    "Failed to execute query stream: {}",
+                    convert_error(&e)
+                ))
            })?;
        Ok(RecordBatchIterator::new(inner_stream))
    }
@@ -198,7 +228,10 @@ impl VectorQuery {
    #[napi]
    pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
        self.inner.explain_plan(verbose).await.map_err(|e| {
-            napi::Error::from_reason(format!("Failed to retrieve the query plan: {}", e))
+            napi::Error::from_reason(format!(
+                "Failed to retrieve the query plan: {}",
+                convert_error(&e)
+            ))
        })
    }
 }
--- a/nodejs/src/table.rs
+++ b/nodejs/src/table.rs
@@ -72,10 +72,7 @@ impl Table {
    /// Return Schema as empty Arrow IPC file.
    #[napi(catch_unwind)]
    pub async fn schema(&self) -> napi::Result<Buffer> {
-        let schema =
-            self.inner_ref()?.schema().await.map_err(|e| {
-                napi::Error::from_reason(format!("Failed to create IPC file: {}", e))
-            })?;
+        let schema = self.inner_ref()?.schema().await.default_error()?;
        let mut writer = FileWriter::try_new(vec![], &schema)
            .map_err(|e| napi::Error::from_reason(format!("Failed to create IPC file: {}", e)))?;
        writer
@@ -100,12 +97,7 @@ impl Table {
            return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
        };

-        op.execute().await.map_err(|e| {
-            napi::Error::from_reason(format!(
-                "Failed to add batches to table {}: {}",
-                self.name, e
-            ))
-        })
+        op.execute().await.default_error()
    }

    #[napi(catch_unwind)]
@@ -114,22 +106,12 @@ impl Table {
            .count_rows(filter)
            .await
            .map(|val| val as i64)
-            .map_err(|e| {
-                napi::Error::from_reason(format!(
-                    "Failed to count rows in table {}: {}",
-                    self.name, e
-                ))
-            })
+            .default_error()
    }

    #[napi(catch_unwind)]
    pub async fn delete(&self, predicate: String) -> napi::Result<()> {
-        self.inner_ref()?.delete(&predicate).await.map_err(|e| {
-            napi::Error::from_reason(format!(
-                "Failed to delete rows in table {}: predicate={}",
-                self.name, e
-            ))
-        })
+        self.inner_ref()?.delete(&predicate).await.default_error()
    }

    #[napi(catch_unwind)]
@@ -187,12 +169,7 @@ impl Table {
        self.inner_ref()?
            .add_columns(transforms, None)
            .await
-            .map_err(|err| {
-                napi::Error::from_reason(format!(
-                    "Failed to add columns to table {}: {}",
-                    self.name, err
-                ))
-            })?;
+            .default_error()?;
        Ok(())
    }

@@ -213,12 +190,7 @@ impl Table {
        self.inner_ref()?
            .alter_columns(&alterations)
            .await
-            .map_err(|err| {
-                napi::Error::from_reason(format!(
-                    "Failed to alter columns in table {}: {}",
-                    self.name, err
-                ))
-            })?;
+            .default_error()?;
        Ok(())
    }

@@ -228,12 +200,7 @@ impl Table {
        self.inner_ref()?
            .drop_columns(&col_refs)
            .await
-            .map_err(|err| {
-                napi::Error::from_reason(format!(
-                    "Failed to drop columns from table {}: {}",
-                    self.name, err
-                ))
-            })?;
+            .default_error()?;
        Ok(())
    }

--- a/python/.bumpversion.toml
+++ b/python/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.14.0"
+current_version = "0.16.0-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.14.0"
+version = "0.16.0-beta.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow = { version = "52.1", features = ["pyarrow"] }
 lancedb = { path = "../rust/lancedb" }
-env_logger = "0.10"
+env_logger.workspace = true
 pyo3 = { version = "0.21", features = ["extension-module", "abi3-py38", "gil-refs"] }
 # Using this fork for now: https://github.com/awestlake87/pyo3-asyncio/issues/119
 # pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -3,14 +3,11 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
    "deprecation",
-    "pylance==0.18.2",
-    "requests>=2.31.0",
-    "retry>=0.9.2",
+    "nest-asyncio~=1.0",
+    "pylance==0.19.2-beta.3",
    "tqdm>=4.27.0",
    "pydantic>=1.10",
-    "attrs>=21.3.0",
    "packaging",
-    "cachetools",
    "overrides>=0.7",
 ]
 description = "lancedb"
@@ -62,6 +59,7 @@ dev = ["ruff", "pre-commit"]
 docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
 embeddings = [
+    "requests>=2.31.0",
    "openai>=1.6.1",
    "sentence-transformers",
    "torch",
--- a/python/python/lancedb/init.py
+++ b/python/python/lancedb/init.py
@@ -19,12 +19,10 @@ from typing import Dict, Optional, Union, Any

 __version__ = importlib.metadata.version("lancedb")

-from lancedb.remote import ClientConfig
-
 from ._lancedb import connect as lancedb_connect
 from .common import URI, sanitize_uri
 from .db import AsyncConnection, DBConnection, LanceDBConnection
-from .remote.db import RemoteDBConnection
+from .remote import ClientConfig
 from .schema import vector
 from .table import AsyncTable

@@ -37,6 +35,7 @@ def connect(
    host_override: Optional[str] = None,
    read_consistency_interval: Optional[timedelta] = None,
    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
+    client_config: Union[ClientConfig, Dict[str, Any], None] = None,
    **kwargs: Any,
 ) -> DBConnection:
    """Connect to a LanceDB database.
@@ -64,14 +63,10 @@ def connect(
        the last check, then the table will be checked for updates. Note: this
        consistency only applies to read operations. Write operations are
        always consistent.
-    request_thread_pool: int or ThreadPoolExecutor, optional
-        The thread pool to use for making batch requests to the LanceDB Cloud API.
-        If an integer, then a ThreadPoolExecutor will be created with that
-        number of threads. If None, then a ThreadPoolExecutor will be created
-        with the default number of threads. If a ThreadPoolExecutor, then that
-        executor will be used for making requests. This is for LanceDB Cloud
-        only and is only used when making batch requests (i.e., passing in
-        multiple queries to the search method at once).
+    client_config: ClientConfig or dict, optional
+        Configuration options for the LanceDB Cloud HTTP client. If a dict, then
+        the keys are the attributes of the ClientConfig class. If None, then the
+        default configuration is used.

    Examples
    --------
@@ -94,6 +89,8 @@ def connect(
    conn : DBConnection
        A connection to a LanceDB database.
    """
+    from .remote.db import RemoteDBConnection
+
    if isinstance(uri, str) and uri.startswith("db://"):
        if api_key is None:
            api_key = os.environ.get("LANCEDB_API_KEY")
@@ -106,7 +103,9 @@ def connect(
            api_key,
            region,
            host_override,
+            # TODO: remove this (deprecation warning downstream)
            request_thread_pool=request_thread_pool,
+            client_config=client_config,
            **kwargs,
        )

--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -36,6 +36,8 @@ class Connection(object):
        data_storage_version: Optional[str] = None,
        enable_v2_manifest_paths: Optional[bool] = None,
    ) -> Table: ...
+    async def rename_table(self, old_name: str, new_name: str) -> None: ...
+    async def drop_table(self, name: str) -> None: ...

 class Table:
    def name(self) -> str: ...
--- a/python/python/lancedb/conftest.py
+++ b/python/python/lancedb/conftest.py
@@ -26,7 +26,7 @@ registry = EmbeddingFunctionRegistry.get_instance()
@registry.register("test")
 class MockTextEmbeddingFunction(TextEmbeddingFunction):
    """
-    Return the hash of the first 10 characters
+    Return the hash of the first 10 characters (normalized)
    """

    def generate_embeddings(self, texts):
@@ -41,6 +41,23 @@ class MockTextEmbeddingFunction(TextEmbeddingFunction):
        return 10


+@registry.register("nonnorm")
+class MockNonNormTextEmbeddingFunction(TextEmbeddingFunction):
+    """
+    Return the ord of the first 10 characters (not normalized)
+    """
+
+    def generate_embeddings(self, texts):
+        return [self._compute_one_embedding(row) for row in texts]
+
+    def _compute_one_embedding(self, row):
+        emb = np.array([float(ord(c)) for c in row[:10]])
+        return emb if len(emb) == 10 else [0] * 10
+
+    def ndims(self):
+        return 10
+
+
 class RateLimitedAPI:
    rate_limit = 0.1  # 1 request per 0.1 second
    last_request_time = 0
--- a/python/python/lancedb/db.py
+++ b/python/python/lancedb/db.py
@@ -817,6 +817,18 @@ class AsyncConnection(object):
        table = await self._inner.open_table(name, storage_options, index_cache_size)
        return AsyncTable(table)

+    async def rename_table(self, old_name: str, new_name: str):
+        """Rename a table in the database.
+
+        Parameters
+        ----------
+        old_name: str
+            The current name of the table.
+        new_name: str
+            The new name of the table.
+        """
+        await self._inner.rename_table(old_name, new_name)
+
    async def drop_table(self, name: str):
        """Drop a table from the database.

--- a/python/python/lancedb/embeddings/jinaai.py
+++ b/python/python/lancedb/embeddings/jinaai.py
@@ -13,7 +13,6 @@

 import os
 import io
-import requests
 import base64
 from urllib.parse import urlparse
 from pathlib import Path
@@ -226,6 +225,8 @@ class JinaEmbeddings(EmbeddingFunction):
        return [result["embedding"] for result in sorted_embeddings]

    def _init_client(self):
+        import requests
+
        if JinaEmbeddings._session is None:
            if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
                api_key_not_found_help("jina")
--- a/python/python/lancedb/embeddings/utils.py
+++ b/python/python/lancedb/embeddings/utils.py
@@ -21,14 +21,35 @@ import time
 import urllib.error
 import weakref
 import logging
+from functools import wraps
 from typing import Callable, List, Union
 import numpy as np
 import pyarrow as pa
 from lance.vector import vec_to_table
-from retry import retry

 from ..util import deprecated, safe_import_pandas

+
+# ruff: noqa: PERF203
+def retry(tries=10, delay=1, max_delay=30, backoff=3, jitter=1):
+    def wrapper(fn):
+        @wraps(fn)
+        def wrapped(*args, **kwargs):
+            for i in range(tries):
+                try:
+                    return fn(*args, **kwargs)
+                except Exception:
+                    if i + 1 == tries:
+                        raise
+                    else:
+                        sleep = min(delay * (backoff**i) + jitter, max_delay)
+                        time.sleep(sleep)
+
+        return wrapped
+
+    return wrapper
+
+
 pd = safe_import_pandas()

 DATA = Union[pa.Table, "pd.DataFrame"]
--- a/python/python/lancedb/index.py
+++ b/python/python/lancedb/index.py
@@ -7,6 +7,27 @@ from ._lancedb import (
    IndexConfig,
 )

+lang_mapping = {
+    "ar": "Arabic",
+    "da": "Danish",
+    "du": "Dutch",
+    "en": "English",
+    "fi": "Finnish",
+    "fr": "French",
+    "de": "German",
+    "gr": "Greek",
+    "hu": "Hungarian",
+    "it": "Italian",
+    "no": "Norwegian",
+    "pt": "Portuguese",
+    "ro": "Romanian",
+    "ru": "Russian",
+    "es": "Spanish",
+    "sv": "Swedish",
+    "ta": "Tamil",
+    "tr": "Turkish",
+}
+

 class BTree:
    """Describes a btree index configuration
@@ -78,7 +99,17 @@ class FTS:
    For example, it works with `title`, `description`, `content`, etc.
    """

-    def __init__(self, with_position: bool = True):
+    def __init__(
+        self,
+        with_position: bool = True,
+        base_tokenizer: str = "simple",
+        language: str = "English",
+        max_token_length: Optional[int] = 40,
+        lower_case: bool = True,
+        stem: bool = False,
+        remove_stop_words: bool = False,
+        ascii_folding: bool = False,
+    ):
        self._inner = LanceDbIndex.fts(with_position=with_position)


@@ -436,6 +467,8 @@ class IvfPq:

            The default value is 256.
        """
+        if distance_type is not None:
+            distance_type = distance_type.lower()
        self._inner = LanceDbIndex.ivf_pq(
            distance_type=distance_type,
            num_partitions=num_partitions,
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -88,6 +88,11 @@ class Query(pydantic.BaseModel):
          tuning advice.
    offset: int
        The offset to start fetching results from
+    fast_search: bool
+        Skip a flat search of unindexed data. This will improve
+        search performance but search results will not include unindexed data.
+
+        - *default False*.
    """

    vector_column: Optional[str] = None
@@ -124,6 +129,8 @@ class Query(pydantic.BaseModel):

    offset: int = 0

+    fast_search: bool = False
+

 class LanceQueryBuilder(ABC):
    """An abstract query builder. Subclasses are defined for vector search,
@@ -139,6 +146,7 @@ class LanceQueryBuilder(ABC):
        vector_column_name: str,
        ordering_field_name: Optional[str] = None,
        fts_columns: Union[str, List[str]] = [],
+        fast_search: bool = False,
    ) -> LanceQueryBuilder:
        """
        Create a query builder based on the given query and query type.
@@ -155,6 +163,8 @@ class LanceQueryBuilder(ABC):
            If "auto", the query type is inferred based on the query.
        vector_column_name: str
            The name of the vector column to use for vector search.
+        fast_search: bool
+            Skip flat search of unindexed data.
        """
        # Check hybrid search first as it supports empty query pattern
        if query_type == "hybrid":
@@ -196,7 +206,9 @@ class LanceQueryBuilder(ABC):
        else:
            raise TypeError(f"Unsupported query type: {type(query)}")

-        return LanceVectorQueryBuilder(table, query, vector_column_name, str_query)
+        return LanceVectorQueryBuilder(
+            table, query, vector_column_name, str_query, fast_search
+        )

    @classmethod
    def _resolve_query(cls, table, query, query_type, vector_column_name):
@@ -469,6 +481,7 @@ class LanceQueryBuilder(ABC):
        >>> plan = table.search(query).explain_plan(True)
        >>> print(plan) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
        ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
+        GlobalLimitExec: skip=0, fetch=10
          FilterExec: _distance@2 IS NOT NULL
            SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
              KNNVectorDistance: metric=l2
@@ -488,7 +501,16 @@ class LanceQueryBuilder(ABC):
            nearest={
                "column": self._vector_column,
                "q": self._query,
+                "k": self._limit,
+                "metric": self._metric,
+                "nprobes": self._nprobes,
+                "refine_factor": self._refine_factor,
            },
+            prefilter=self._prefilter,
+            filter=self._str_query,
+            limit=self._limit,
+            with_row_id=self._with_row_id,
+            offset=self._offset,
        ).explain_plan(verbose)

    def vector(self, vector: Union[np.ndarray, list]) -> LanceQueryBuilder:
@@ -565,6 +587,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
        query: Union[np.ndarray, list, "PIL.Image.Image"],
        vector_column: str,
        str_query: Optional[str] = None,
+        fast_search: bool = False,
    ):
        super().__init__(table)
        self._query = query
@@ -575,6 +598,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
        self._prefilter = False
        self._reranker = None
        self._str_query = str_query
+        self._fast_search = fast_search

    def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceVectorQueryBuilder:
        """Set the distance metric to use.
@@ -675,6 +699,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
            vector_column=self._vector_column,
            with_row_id=self._with_row_id,
            offset=self._offset,
+            fast_search=self._fast_search,
        )
        result_set = self._table._execute_query(query, batch_size)
        if self._reranker is not None:
@@ -968,6 +993,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
        self._reranker = RRFReranker()
        self._nprobes = None
        self._refine_factor = None
+        self._metric = None
        self._phrase_query = False

    def _validate_query(self, query, vector=None, text=None):
@@ -1035,6 +1061,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
            self._fts_query.with_row_id(True)
        if self._phrase_query:
            self._fts_query.phrase_query(True)
+        if self._metric:
+            self._vector_query.metric(self._metric)
        if self._nprobes:
            self._vector_query.nprobes(self._nprobes)
        if self._refine_factor:
@@ -1052,6 +1080,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
        if self._norm == "rank":
            vector_results = self._rank(vector_results, "_distance")
            fts_results = self._rank(fts_results, "_score")
+
        # normalize the scores to be between 0 and 1, 0 being most relevant
        vector_results = self._normalize_scores(vector_results, "_distance")

@@ -1100,7 +1129,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
            rng = max
        else:
            rng = max - min
-        scores = (scores - min) / rng
+        # If rng is 0 then min and max are both 0 and so we can leave the scores as is
+        if rng != 0:
+            scores = (scores - min) / rng
        if invert:
            scores = 1 - scores
        # replace the _score column with the ranks
@@ -1162,6 +1193,22 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
        self._nprobes = nprobes
        return self

+    def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceHybridQueryBuilder:
+        """Set the distance metric to use.
+
+        Parameters
+        ----------
+        metric: "L2" or "cosine" or "dot"
+            The distance metric to use. By default "L2" is used.
+
+        Returns
+        -------
+        LanceVectorQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._metric = metric.lower()
+        return self
+
    def refine_factor(self, refine_factor: int) -> LanceHybridQueryBuilder:
        """
        Refine the vector search results by reading extra elements and
@@ -1278,6 +1325,48 @@ class AsyncQueryBase(object):
        self._inner.offset(offset)
        return self

+    def fast_search(self) -> AsyncQuery:
+        """
+        Skip searching un-indexed data.
+
+        This can make queries faster, but will miss any data that has not been
+        indexed.
+
+        !!! tip
+            You can add new data into an existing index by calling
+            [AsyncTable.optimize][lancedb.table.AsyncTable.optimize].
+        """
+        self._inner.fast_search()
+        return self
+
+    def with_row_id(self) -> AsyncQuery:
+        """
+        Include the _rowid column in the results.
+        """
+        self._inner.with_row_id()
+        return self
+
+    def postfilter(self) -> AsyncQuery:
+        """
+        If this is called then filtering will happen after the search instead of
+        before.
+        By default filtering will be performed before the search.  This is how
+        filtering is typically understood to work.  This prefilter step does add some
+        additional latency.  Creating a scalar index on the filter column(s) can
+        often improve this latency.  However, sometimes a filter is too complex or
+        scalar indices cannot be applied to the column.  In these cases postfiltering
+        can be used instead of prefiltering to improve latency.
+        Post filtering applies the filter to the results of the search.  This
+        means we only run the filter on a much smaller set of data.  However, it can
+        cause the query to return fewer than `limit` results (or even no results) if
+        none of the nearest results match the filter.
+        Post filtering happens during the "refine stage" (described in more detail in
+        @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
+        factor can often help restore some of the results lost by post filtering.
+        """
+        self._inner.postfilter()
+        return self
+
    async def to_batches(
        self, *, max_batch_length: Optional[int] = None
    ) -> AsyncRecordBatchReader:
@@ -1581,30 +1670,6 @@ class AsyncVectorQuery(AsyncQueryBase):
        self._inner.distance_type(distance_type)
        return self

-    def postfilter(self) -> AsyncVectorQuery:
-        """
-        If this is called then filtering will happen after the vector search instead of
-        before.
-
-        By default filtering will be performed before the vector search.  This is how
-        filtering is typically understood to work.  This prefilter step does add some
-        additional latency.  Creating a scalar index on the filter column(s) can
-        often improve this latency.  However, sometimes a filter is too complex or
-        scalar indices cannot be applied to the column.  In these cases postfiltering
-        can be used instead of prefiltering to improve latency.
-
-        Post filtering applies the filter to the results of the vector search.  This
-        means we only run the filter on a much smaller set of data.  However, it can
-        cause the query to return fewer than `limit` results (or even no results) if
-        none of the nearest results match the filter.
-
-        Post filtering happens during the "refine stage" (described in more detail in
-        @see {@link VectorQuery#refineFactor}).  This means that setting a higher refine
-        factor can often help restore some of the results lost by post filtering.
-        """
-        self._inner.postfilter()
-        return self
-
    def bypass_vector_index(self) -> AsyncVectorQuery:
        """
        If this is called then any vector index is skipped
--- a/python/python/lancedb/remote/init.py
+++ b/python/python/lancedb/remote/init.py
@@ -11,60 +11,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

-import abc
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import timedelta
 from typing import List, Optional

-import attrs
 from lancedb import __version__
-import pyarrow as pa
-from pydantic import BaseModel

-from lancedb.common import VECTOR_COLUMN_NAME
-
-__all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
-
-
-class VectorQuery(BaseModel):
-    # vector to search for
-    vector: List[float]
-
-    # sql filter to refine the query with
-    filter: Optional[str] = None
-
-    # top k results to return
-    k: int
-
-    # # metrics
-    _metric: str = "L2"
-
-    # which columns to return in the results
-    columns: Optional[List[str]] = None
-
-    # optional query parameters for tuning the results,
-    # e.g. `{"nprobes": "10", "refine_factor": "10"}`
-    nprobes: int = 10
-
-    refine_factor: Optional[int] = None
-
-    vector_column: str = VECTOR_COLUMN_NAME
-
-
-@attrs.define
-class VectorQueryResult:
-    # for now the response is directly seralized into a pandas dataframe
-    tbl: pa.Table
-
-    def to_arrow(self) -> pa.Table:
-        return self.tbl
-
-
-class LanceDBClient(abc.ABC):
-    @abc.abstractmethod
-    def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
-        """Query the LanceDB server for the given table and query."""
-        pass
+__all__ = ["TimeoutConfig", "RetryConfig", "ClientConfig"]


@dataclass
@@ -163,8 +116,8 @@ class RetryConfig:
@dataclass
 class ClientConfig:
    user_agent: str = f"LanceDB-Python-Client/{__version__}"
-    retry_config: Optional[RetryConfig] = None
-    timeout_config: Optional[TimeoutConfig] = None
+    retry_config: RetryConfig = field(default_factory=RetryConfig)
+    timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig)

    def __post_init__(self):
        if isinstance(self.retry_config, dict):
--- a/python/python/lancedb/remote/arrow.py
+++ b/python/python/lancedb/remote/arrow.py
@@ -1,25 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-from typing import Iterable, Union
-import pyarrow as pa
-
-
-def to_ipc_binary(table: Union[pa.Table, Iterable[pa.RecordBatch]]) -> bytes:
-    """Serialize a PyArrow Table to IPC binary."""
-    sink = pa.BufferOutputStream()
-    if isinstance(table, Iterable):
-        table = pa.Table.from_batches(table)
-    with pa.ipc.new_stream(sink, table.schema) as writer:
-        writer.write_table(table)
-    return sink.getvalue().to_pybytes()
--- a/python/python/lancedb/remote/client.py
+++ b/python/python/lancedb/remote/client.py
@@ -1,269 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-
-import functools
-import logging
-import os
-from typing import Any, Callable, Dict, List, Optional, Union
-from urllib.parse import urljoin
-
-import attrs
-import pyarrow as pa
-import requests
-from pydantic import BaseModel
-from requests.adapters import HTTPAdapter
-from urllib3 import Retry
-
-from lancedb.common import Credential
-from lancedb.remote import VectorQuery, VectorQueryResult
-from lancedb.remote.connection_timeout import LanceDBClientHTTPAdapterFactory
-from lancedb.remote.errors import LanceDBClientError
-
-ARROW_STREAM_CONTENT_TYPE = "application/vnd.apache.arrow.stream"
-
-
-def _check_not_closed(f):
-    @functools.wraps(f)
-    def wrapped(self, *args, **kwargs):
-        if self.closed:
-            raise ValueError("Connection is closed")
-        return f(self, *args, **kwargs)
-
-    return wrapped
-
-
-def _read_ipc(resp: requests.Response) -> pa.Table:
-    resp_body = resp.content
-    with pa.ipc.open_file(pa.BufferReader(resp_body)) as reader:
-        return reader.read_all()
-
-
-@attrs.define(slots=False)
-class RestfulLanceDBClient:
-    db_name: str
-    region: str
-    api_key: Credential
-    host_override: Optional[str] = attrs.field(default=None)
-
-    closed: bool = attrs.field(default=False, init=False)
-
-    connection_timeout: float = attrs.field(default=120.0, kw_only=True)
-    read_timeout: float = attrs.field(default=300.0, kw_only=True)
-
-    @functools.cached_property
-    def session(self) -> requests.Session:
-        sess = requests.Session()
-
-        retry_adapter_instance = retry_adapter(retry_adapter_options())
-        sess.mount(urljoin(self.url, "/v1/table/"), retry_adapter_instance)
-
-        adapter_class = LanceDBClientHTTPAdapterFactory()
-        sess.mount("https://", adapter_class())
-        return sess
-
-    @property
-    def url(self) -> str:
-        return (
-            self.host_override
-            or f"https://{self.db_name}.{self.region}.api.lancedb.com"
-        )
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.close()
-        return False  # Do not suppress exceptions
-
-    def close(self):
-        self.session.close()
-        self.closed = True
-
-    @functools.cached_property
-    def headers(self) -> Dict[str, str]:
-        headers = {
-            "x-api-key": self.api_key,
-        }
-        if self.region == "local":  # Local test mode
-            headers["Host"] = f"{self.db_name}.{self.region}.api.lancedb.com"
-        if self.host_override:
-            headers["x-lancedb-database"] = self.db_name
-        return headers
-
-    @staticmethod
-    def _check_status(resp: requests.Response):
-        # Leaving request id empty for now, as we'll be replacing this impl
-        # with the Rust one shortly.
-        if resp.status_code == 404:
-            raise LanceDBClientError(
-                f"Not found: {resp.text}", request_id="", status_code=404
-            )
-        elif 400 <= resp.status_code < 500:
-            raise LanceDBClientError(
-                f"Bad Request: {resp.status_code}, error: {resp.text}",
-                request_id="",
-                status_code=resp.status_code,
-            )
-        elif 500 <= resp.status_code < 600:
-            raise LanceDBClientError(
-                f"Internal Server Error: {resp.status_code}, error: {resp.text}",
-                request_id="",
-                status_code=resp.status_code,
-            )
-        elif resp.status_code != 200:
-            raise LanceDBClientError(
-                f"Unknown Error: {resp.status_code}, error: {resp.text}",
-                request_id="",
-                status_code=resp.status_code,
-            )
-
-    @_check_not_closed
-    def get(self, uri: str, params: Union[Dict[str, Any], BaseModel] = None):
-        """Send a GET request and returns the deserialized response payload."""
-        if isinstance(params, BaseModel):
-            params: Dict[str, Any] = params.dict(exclude_none=True)
-        with self.session.get(
-            urljoin(self.url, uri),
-            params=params,
-            headers=self.headers,
-            timeout=(self.connection_timeout, self.read_timeout),
-        ) as resp:
-            self._check_status(resp)
-            return resp.json()
-
-    @_check_not_closed
-    def post(
-        self,
-        uri: str,
-        data: Optional[Union[Dict[str, Any], BaseModel, bytes]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        content_type: Optional[str] = None,
-        deserialize: Callable = lambda resp: resp.json(),
-        request_id: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        """Send a POST request and returns the deserialized response payload.
-
-        Parameters
-        ----------
-        uri : str
-            The uri to send the POST request to.
-        data: Union[Dict[str, Any], BaseModel]
-        request_id: Optional[str]
-            Optional client side request id to be sent in the request headers.
-
-        """
-        if isinstance(data, BaseModel):
-            data: Dict[str, Any] = data.dict(exclude_none=True)
-        if isinstance(data, bytes):
-            req_kwargs = {"data": data}
-        else:
-            req_kwargs = {"json": data}
-
-        headers = self.headers.copy()
-        if content_type is not None:
-            headers["content-type"] = content_type
-        if request_id is not None:
-            headers["x-request-id"] = request_id
-        with self.session.post(
-            urljoin(self.url, uri),
-            headers=headers,
-            params=params,
-            timeout=(self.connection_timeout, self.read_timeout),
-            **req_kwargs,
-        ) as resp:
-            self._check_status(resp)
-            return deserialize(resp)
-
-    @_check_not_closed
-    def list_tables(self, limit: int, page_token: Optional[str] = None) -> List[str]:
-        """List all tables in the database."""
-        if page_token is None:
-            page_token = ""
-        json = self.get("/v1/table/", {"limit": limit, "page_token": page_token})
-        return json["tables"]
-
-    @_check_not_closed
-    def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
-        """Query a table."""
-        tbl = self.post(f"/v1/table/{table_name}/query/", query, deserialize=_read_ipc)
-        return VectorQueryResult(tbl)
-
-    def mount_retry_adapter_for_table(self, table_name: str) -> None:
-        """
-        Adds an http adapter to session that will retry retryable requests to the table.
-        """
-        retry_options = retry_adapter_options(methods=["GET", "POST"])
-        retry_adapter_instance = retry_adapter(retry_options)
-        session = self.session
-
-        session.mount(
-            urljoin(self.url, f"/v1/table/{table_name}/query/"), retry_adapter_instance
-        )
-        session.mount(
-            urljoin(self.url, f"/v1/table/{table_name}/describe/"),
-            retry_adapter_instance,
-        )
-        session.mount(
-            urljoin(self.url, f"/v1/table/{table_name}/index/list/"),
-            retry_adapter_instance,
-        )
-
-
-def retry_adapter_options(methods=["GET"]) -> Dict[str, Any]:
-    return {
-        "retries": int(os.environ.get("LANCE_CLIENT_MAX_RETRIES", "3")),
-        "connect_retries": int(os.environ.get("LANCE_CLIENT_CONNECT_RETRIES", "3")),
-        "read_retries": int(os.environ.get("LANCE_CLIENT_READ_RETRIES", "3")),
-        "backoff_factor": float(
-            os.environ.get("LANCE_CLIENT_RETRY_BACKOFF_FACTOR", "0.25")
-        ),
-        "backoff_jitter": float(
-            os.environ.get("LANCE_CLIENT_RETRY_BACKOFF_JITTER", "0.25")
-        ),
-        "statuses": [
-            int(i.strip())
-            for i in os.environ.get(
-                "LANCE_CLIENT_RETRY_STATUSES", "429, 500, 502, 503"
-            ).split(",")
-        ],
-        "methods": methods,
-    }
-
-
-def retry_adapter(options: Dict[str, Any]) -> HTTPAdapter:
-    total_retries = options["retries"]
-    connect_retries = options["connect_retries"]
-    read_retries = options["read_retries"]
-    backoff_factor = options["backoff_factor"]
-    backoff_jitter = options["backoff_jitter"]
-    statuses = options["statuses"]
-    methods = frozenset(options["methods"])
-    logging.debug(
-        f"Setting up retry adapter with {total_retries} retries,"  # noqa G003
-        + f"connect retries {connect_retries}, read retries {read_retries},"
-        + f"backoff factor {backoff_factor}, statuses {statuses}, "
-        + f"methods {methods}"
-    )
-
-    return HTTPAdapter(
-        max_retries=Retry(
-            total=total_retries,
-            connect=connect_retries,
-            read=read_retries,
-            backoff_factor=backoff_factor,
-            backoff_jitter=backoff_jitter,
-            status_forcelist=statuses,
-            allowed_methods=methods,
-        )
-    )
--- a/python/python/lancedb/remote/connection_timeout.py
+++ b/python/python/lancedb/remote/connection_timeout.py
@@ -1,115 +0,0 @@
-#  Copyright 2024 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-# This module contains an adapter that will close connections if they have not been
-# used before a certain timeout. This is necessary because some load balancers will
-# close connections after a certain amount of time, but the request module may not yet
-# have received the FIN/ACK and will try to reuse the connection.
-#
-# TODO some of the code here can be simplified if/when this PR is merged:
-# https://github.com/urllib3/urllib3/pull/3275
-
-import datetime
-import logging
-import os
-
-from requests.adapters import HTTPAdapter
-from urllib3.connection import HTTPSConnection
-from urllib3.connectionpool import HTTPSConnectionPool
-from urllib3.poolmanager import PoolManager
-
-
-def get_client_connection_timeout() -> int:
-    return int(os.environ.get("LANCE_CLIENT_CONNECTION_TIMEOUT", "300"))
-
-
-class LanceDBHTTPSConnection(HTTPSConnection):
-    """
-    HTTPSConnection that tracks the last time it was used.
-    """
-
-    idle_timeout: datetime.timedelta
-    last_activity: datetime.datetime
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.last_activity = datetime.datetime.now()
-
-    def request(self, *args, **kwargs):
-        self.last_activity = datetime.datetime.now()
-        super().request(*args, **kwargs)
-
-    def is_expired(self):
-        return datetime.datetime.now() - self.last_activity > self.idle_timeout
-
-
-def LanceDBHTTPSConnectionPoolFactory(client_idle_timeout: int):
-    """
-    Creates a connection pool class that can be used to close idle connections.
-    """
-
-    class LanceDBHTTPSConnectionPool(HTTPSConnectionPool):
-        # override the connection class
-        ConnectionCls = LanceDBHTTPSConnection
-
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-
-        def _get_conn(self, timeout: float | None = None):
-            logging.debug("Getting https connection")
-            conn = super()._get_conn(timeout)
-            if conn.is_expired():
-                logging.debug("Closing expired connection")
-                conn.close()
-
-            return conn
-
-        def _new_conn(self):
-            conn = super()._new_conn()
-            conn.idle_timeout = datetime.timedelta(seconds=client_idle_timeout)
-            return conn
-
-    return LanceDBHTTPSConnectionPool
-
-
-class LanceDBClientPoolManager(PoolManager):
-    def __init__(
-        self, client_idle_timeout: int, num_pools: int, maxsize: int, **kwargs
-    ):
-        super().__init__(num_pools=num_pools, maxsize=maxsize, **kwargs)
-        # inject our connection pool impl
-        connection_pool_class = LanceDBHTTPSConnectionPoolFactory(
-            client_idle_timeout=client_idle_timeout
-        )
-        self.pool_classes_by_scheme["https"] = connection_pool_class
-
-
-def LanceDBClientHTTPAdapterFactory():
-    """
-    Creates an HTTPAdapter class that can be used to close idle connections
-    """
-
-    # closure over the timeout
-    client_idle_timeout = get_client_connection_timeout()
-
-    class LanceDBClientRequestHTTPAdapter(HTTPAdapter):
-        def init_poolmanager(self, connections, maxsize, block=False):
-            # inject our pool manager impl
-            self.poolmanager = LanceDBClientPoolManager(
-                client_idle_timeout=client_idle_timeout,
-                num_pools=connections,
-                maxsize=maxsize,
-                block=block,
-            )
-
-    return LanceDBClientRequestHTTPAdapter
--- a/python/python/lancedb/remote/db.py
+++ b/python/python/lancedb/remote/db.py
@@ -11,13 +11,16 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+import asyncio
+from datetime import timedelta
 import logging
-import uuid
 from concurrent.futures import ThreadPoolExecutor
-from typing import Iterable, List, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Union
 from urllib.parse import urlparse
+import warnings

-from cachetools import TTLCache
+from lancedb import connect_async
+from lancedb.remote import ClientConfig
 import pyarrow as pa
 from overrides import override

@@ -25,10 +28,8 @@ from ..common import DATA
 from ..db import DBConnection
 from ..embeddings import EmbeddingFunctionConfig
 from ..pydantic import LanceModel
-from ..table import Table, sanitize_create_table
+from ..table import Table
 from ..util import validate_table_name
-from .arrow import to_ipc_binary
-from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient


 class RemoteDBConnection(DBConnection):
@@ -41,26 +42,70 @@ class RemoteDBConnection(DBConnection):
        region: str,
        host_override: Optional[str] = None,
        request_thread_pool: Optional[ThreadPoolExecutor] = None,
-        connection_timeout: float = 120.0,
-        read_timeout: float = 300.0,
+        client_config: Union[ClientConfig, Dict[str, Any], None] = None,
+        connection_timeout: Optional[float] = None,
+        read_timeout: Optional[float] = None,
    ):
        """Connect to a remote LanceDB database."""
+
+        if isinstance(client_config, dict):
+            client_config = ClientConfig(**client_config)
+        elif client_config is None:
+            client_config = ClientConfig()
+
+        # These are legacy options from the old Python-based client. We keep them
+        # here for backwards compatibility, but will remove them in a future release.
+        if request_thread_pool is not None:
+            warnings.warn(
+                "request_thread_pool is no longer used and will be removed in "
+                "a future release.",
+                DeprecationWarning,
+            )
+
+        if connection_timeout is not None:
+            warnings.warn(
+                "connection_timeout is deprecated and will be removed in a future "
+                "release. Please use client_config.timeout_config.connect_timeout "
+                "instead.",
+                DeprecationWarning,
+            )
+            client_config.timeout_config.connect_timeout = timedelta(
+                seconds=connection_timeout
+            )
+
+        if read_timeout is not None:
+            warnings.warn(
+                "read_timeout is deprecated and will be removed in a future release. "
+                "Please use client_config.timeout_config.read_timeout instead.",
+                DeprecationWarning,
+            )
+            client_config.timeout_config.read_timeout = timedelta(seconds=read_timeout)
+
        parsed = urlparse(db_url)
        if parsed.scheme != "db":
            raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
-        self._uri = str(db_url)
        self.db_name = parsed.netloc
-        self.api_key = api_key
-        self._client = RestfulLanceDBClient(
-            self.db_name,
-            region,
-            api_key,
-            host_override,
-            connection_timeout=connection_timeout,
-            read_timeout=read_timeout,
+
+        import nest_asyncio
+
+        nest_asyncio.apply()
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._loop)
+
+        self.client_config = client_config
+
+        self._conn = self._loop.run_until_complete(
+            connect_async(
+                db_url,
+                api_key=api_key,
+                region=region,
+                host_override=host_override,
+                client_config=client_config,
+            )
        )
-        self._request_thread_pool = request_thread_pool
-        self._table_cache = TTLCache(maxsize=10000, ttl=300)

    def __repr__(self) -> str:
        return f"RemoteConnect(name={self.db_name})"
@@ -82,16 +127,9 @@ class RemoteDBConnection(DBConnection):
        -------
        An iterator of table names.
        """
-        while True:
-            result = self._client.list_tables(limit, page_token)
-
-            if len(result) > 0:
-                page_token = result[len(result) - 1]
-            else:
-                break
-            for item in result:
-                self._table_cache[item] = True
-                yield item
+        return self._loop.run_until_complete(
+            self._conn.table_names(start_after=page_token, limit=limit)
+        )

    @override
    def open_table(self, name: str, *, index_cache_size: Optional[int] = None) -> Table:
@@ -108,20 +146,14 @@ class RemoteDBConnection(DBConnection):
        """
        from .table import RemoteTable

-        self._client.mount_retry_adapter_for_table(name)
-
        if index_cache_size is not None:
            logging.info(
                "index_cache_size is ignored in LanceDb Cloud"
                " (there is no local cache to configure)"
            )

-        # check if table exists
-        if self._table_cache.get(name) is None:
-            self._client.post(f"/v1/table/{name}/describe/")
-            self._table_cache[name] = True
-
-        return RemoteTable(self, name)
+        table = self._loop.run_until_complete(self._conn.open_table(name))
+        return RemoteTable(table, self.db_name, self._loop)

    @override
    def create_table(
@@ -233,27 +265,20 @@ class RemoteDBConnection(DBConnection):
                "Please vote https://github.com/lancedb/lancedb/issues/626 "
                "for this feature."
            )
-        if mode is not None:
-            logging.warning("mode is not yet supported on LanceDB Cloud.")
-
-        data, schema = sanitize_create_table(
-            data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
-        )

        from .table import RemoteTable

-        data = to_ipc_binary(data)
-        request_id = uuid.uuid4().hex
-
-        self._client.post(
-            f"/v1/table/{name}/create/",
-            data=data,
-            request_id=request_id,
-            content_type=ARROW_STREAM_CONTENT_TYPE,
+        table = self._loop.run_until_complete(
+            self._conn.create_table(
+                name,
+                data,
+                mode=mode,
+                schema=schema,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
+            )
        )
-
-        self._table_cache[name] = True
-        return RemoteTable(self, name)
+        return RemoteTable(table, self.db_name, self._loop)

    @override
    def drop_table(self, name: str):
@@ -264,11 +289,7 @@ class RemoteDBConnection(DBConnection):
        name: str
            The name of the table.
        """
-
-        self._client.post(
-            f"/v1/table/{name}/drop/",
-        )
-        self._table_cache.pop(name, default=None)
+        self._loop.run_until_complete(self._conn.drop_table(name))

    @override
    def rename_table(self, cur_name: str, new_name: str):
@@ -281,12 +302,7 @@ class RemoteDBConnection(DBConnection):
        new_name: str
            The new name of the table.
        """
-        self._client.post(
-            f"/v1/table/{cur_name}/rename/",
-            data={"new_table_name": new_name},
-        )
-        self._table_cache.pop(cur_name, default=None)
-        self._table_cache[new_name] = True
+        self._loop.run_until_complete(self._conn.rename_table(cur_name, new_name))

    async def close(self):
        """Close the connection to the database."""
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -11,53 +11,56 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+import asyncio
 import logging
-import uuid
-from concurrent.futures import Future
 from functools import cached_property
 from typing import Dict, Iterable, List, Optional, Union, Literal

+from lancedb.index import FTS, BTree, Bitmap, HnswPq, HnswSq, IvfPq, LabelList
 import pyarrow as pa
-from lance import json_to_schema

 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 from lancedb.merge import LanceMergeInsertBuilder
 from lancedb.embeddings import EmbeddingFunctionRegistry

 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder
-from ..table import Query, Table, _sanitize_data
-from ..util import value_to_sql, infer_vector_column_name
-from .arrow import to_ipc_binary
-from .client import ARROW_STREAM_CONTENT_TYPE
-from .db import RemoteDBConnection
+from ..table import AsyncTable, Query, Table


 class RemoteTable(Table):
-    def __init__(self, conn: RemoteDBConnection, name: str):
-        self._conn = conn
-        self.name = name
+    def __init__(
+        self,
+        table: AsyncTable,
+        db_name: str,
+        loop: Optional[asyncio.AbstractEventLoop] = None,
+    ):
+        self._loop = loop
+        self._table = table
+        self.db_name = db_name
+
+    @property
+    def name(self) -> str:
+        """The name of the table"""
+        return self._table.name

    def __repr__(self) -> str:
-        return f"RemoteTable({self._conn.db_name}.{self.name})"
+        return f"RemoteTable({self.db_name}.{self.name})"

    def __len__(self) -> int:
        self.count_rows(None)

-    @cached_property
+    @property
    def schema(self) -> pa.Schema:
        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
        of this Table

        """
-        resp = self._conn._client.post(f"/v1/table/{self.name}/describe/")
-        schema = json_to_schema(resp["schema"])
-        return schema
+        return self._loop.run_until_complete(self._table.schema())

    @property
    def version(self) -> int:
        """Get the current version of the table"""
-        resp = self._conn._client.post(f"/v1/table/{self.name}/describe/")
-        return resp["version"]
+        return self._loop.run_until_complete(self._table.version())

    @cached_property
    def embedding_functions(self) -> dict:
@@ -84,20 +87,18 @@ class RemoteTable(Table):

    def list_indices(self):
        """List all the indices on the table"""
-        resp = self._conn._client.post(f"/v1/table/{self.name}/index/list/")
-        return resp
+        return self._loop.run_until_complete(self._table.list_indices())

    def index_stats(self, index_uuid: str):
        """List all the stats of a specified index"""
-        resp = self._conn._client.post(
-            f"/v1/table/{self.name}/index/{index_uuid}/stats/"
-        )
-        return resp
+        return self._loop.run_until_complete(self._table.index_stats(index_uuid))

    def create_scalar_index(
        self,
        column: str,
        index_type: Literal["BTREE", "BITMAP", "LABEL_LIST", "scalar"] = "scalar",
+        *,
+        replace: bool = False,
    ):
        """Creates a scalar index
        Parameters
@@ -107,20 +108,23 @@ class RemoteTable(Table):
            or string column.
        index_type : str
            The index type of the scalar index. Must be "scalar" (BTREE),
-            "BTREE", "BITMAP", or "LABEL_LIST"
+            "BTREE", "BITMAP", or "LABEL_LIST",
+        replace : bool
+            If True, replace the existing index with the new one.
        """
+        if index_type == "scalar" or index_type == "BTREE":
+            config = BTree()
+        elif index_type == "BITMAP":
+            config = Bitmap()
+        elif index_type == "LABEL_LIST":
+            config = LabelList()
+        else:
+            raise ValueError(f"Unknown index type: {index_type}")

-        data = {
-            "column": column,
-            "index_type": index_type,
-            "replace": True,
-        }
-        resp = self._conn._client.post(
-            f"/v1/table/{self.name}/create_scalar_index/", data=data
+        self._loop.run_until_complete(
+            self._table.create_index(column, config=config, replace=replace)
        )

-        return resp
-
    def create_fts_index(
        self,
        column: str,
@@ -128,15 +132,10 @@ class RemoteTable(Table):
        replace: bool = False,
        with_position: bool = True,
    ):
-        data = {
-            "column": column,
-            "index_type": "FTS",
-            "replace": replace,
-        }
-        resp = self._conn._client.post(
-            f"/v1/table/{self.name}/create_index/", data=data
+        config = FTS(with_position=with_position)
+        self._loop.run_until_complete(
+            self._table.create_index(column, config=config, replace=replace)
        )
-        return resp

    def create_index(
        self,
@@ -204,17 +203,22 @@ class RemoteTable(Table):
                "Existing indexes will always be replaced."
            )

-        data = {
-            "column": vector_column_name,
-            "index_type": index_type,
-            "metric_type": metric,
-            "index_cache_size": index_cache_size,
-        }
-        resp = self._conn._client.post(
-            f"/v1/table/{self.name}/create_index/", data=data
-        )
+        index_type = index_type.upper()
+        if index_type == "VECTOR" or index_type == "IVF_PQ":
+            config = IvfPq(distance_type=metric)
+        elif index_type == "IVF_HNSW_PQ":
+            config = HnswPq(distance_type=metric)
+        elif index_type == "IVF_HNSW_SQ":
+            config = HnswSq(distance_type=metric)
+        else:
+            raise ValueError(
+                f"Unknown vector index type: {index_type}. Valid options are"
+                " 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
+            )

-        return resp
+        self._loop.run_until_complete(
+            self._table.create_index(vector_column_name, config=config)
+        )

    def add(
        self,
@@ -246,22 +250,10 @@ class RemoteTable(Table):
            The value to use when filling vectors. Only used if on_bad_vectors="fill".

        """
-        data, _ = _sanitize_data(
-            data,
-            self.schema,
-            metadata=self.schema.metadata,
-            on_bad_vectors=on_bad_vectors,
-            fill_value=fill_value,
-        )
-        payload = to_ipc_binary(data)
-
-        request_id = uuid.uuid4().hex
-
-        self._conn._client.post(
-            f"/v1/table/{self.name}/insert/",
-            data=payload,
-            params={"request_id": request_id, "mode": mode},
-            content_type=ARROW_STREAM_CONTENT_TYPE,
+        self._loop.run_until_complete(
+            self._table.add(
+                data, mode=mode, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            )
        )

    def search(
@@ -270,6 +262,7 @@ class RemoteTable(Table):
        vector_column_name: Optional[str] = None,
        query_type="auto",
        fts_columns: Optional[Union[str, List[str]]] = None,
+        fast_search: bool = False,
    ) -> LanceVectorQueryBuilder:
        """Create a search query to find the nearest neighbors
        of the given query vector. We currently support [vector search][search]
@@ -314,6 +307,12 @@ class RemoteTable(Table):
            - If the table has multiple vector columns then the *vector_column_name*
            needs to be specified. Otherwise, an error is raised.

+        fast_search: bool, optional
+            Skip a flat search of unindexed data. This may improve
+            search performance but search results will not include unindexed data.
+
+            - *default False*.
+
        Returns
        -------
        LanceQueryBuilder
@@ -330,12 +329,6 @@ class RemoteTable(Table):
        # empty query builder is not supported in saas, raise error
        if query is None and query_type != "hybrid":
            raise ValueError("Empty query is not supported")
-        vector_column_name = infer_vector_column_name(
-            schema=self.schema,
-            query_type=query_type,
-            query=query,
-            vector_column_name=vector_column_name,
-        )

        return LanceQueryBuilder.create(
            self,
@@ -343,42 +336,15 @@ class RemoteTable(Table):
            query_type,
            vector_column_name=vector_column_name,
            fts_columns=fts_columns,
+            fast_search=fast_search,
        )

    def _execute_query(
        self, query: Query, batch_size: Optional[int] = None
    ) -> pa.RecordBatchReader:
-        if (
-            query.vector is not None
-            and len(query.vector) > 0
-            and not isinstance(query.vector[0], float)
-        ):
-            if self._conn._request_thread_pool is None:
-
-                def submit(name, q):
-                    f = Future()
-                    f.set_result(self._conn._client.query(name, q))
-                    return f
-
-            else:
-
-                def submit(name, q):
-                    return self._conn._request_thread_pool.submit(
-                        self._conn._client.query, name, q
-                    )
-
-            results = []
-            for v in query.vector:
-                v = list(v)
-                q = query.copy()
-                q.vector = v
-                results.append(submit(self.name, q))
-            return pa.concat_tables(
-                [add_index(r.result().to_arrow(), i) for i, r in enumerate(results)]
-            ).to_reader()
-        else:
-            result = self._conn._client.query(self.name, query)
-            return result.to_arrow().to_reader()
+        return self._loop.run_until_complete(
+            self._table._execute_query(query, batch_size=batch_size)
+        )

    def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
        """Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder]
@@ -395,42 +361,8 @@ class RemoteTable(Table):
        on_bad_vectors: str,
        fill_value: float,
    ):
-        data, _ = _sanitize_data(
-            new_data,
-            self.schema,
-            metadata=None,
-            on_bad_vectors=on_bad_vectors,
-            fill_value=fill_value,
-        )
-        payload = to_ipc_binary(data)
-
-        params = {}
-        if len(merge._on) != 1:
-            raise ValueError(
-                "RemoteTable only supports a single on key in merge_insert"
-            )
-        params["on"] = merge._on[0]
-        params["when_matched_update_all"] = str(merge._when_matched_update_all).lower()
-        if merge._when_matched_update_all_condition is not None:
-            params["when_matched_update_all_filt"] = (
-                merge._when_matched_update_all_condition
-            )
-        params["when_not_matched_insert_all"] = str(
-            merge._when_not_matched_insert_all
-        ).lower()
-        params["when_not_matched_by_source_delete"] = str(
-            merge._when_not_matched_by_source_delete
-        ).lower()
-        if merge._when_not_matched_by_source_condition is not None:
-            params["when_not_matched_by_source_delete_filt"] = (
-                merge._when_not_matched_by_source_condition
-            )
-
-        self._conn._client.post(
-            f"/v1/table/{self.name}/merge_insert/",
-            data=payload,
-            params=params,
-            content_type=ARROW_STREAM_CONTENT_TYPE,
+        self._loop.run_until_complete(
+            self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
        )

    def delete(self, predicate: str):
@@ -480,8 +412,7 @@ class RemoteTable(Table):
           x      vector  _distance # doctest: +SKIP
        0  2  [3.0, 4.0]       85.0 # doctest: +SKIP
        """
-        payload = {"predicate": predicate}
-        self._conn._client.post(f"/v1/table/{self.name}/delete/", data=payload)
+        self._loop.run_until_complete(self._table.delete(predicate))

    def update(
        self,
@@ -531,18 +462,9 @@ class RemoteTable(Table):
        2  2  [10.0, 10.0] # doctest: +SKIP

        """
-        if values is not None and values_sql is not None:
-            raise ValueError("Only one of values or values_sql can be provided")
-        if values is None and values_sql is None:
-            raise ValueError("Either values or values_sql must be provided")
-
-        if values is not None:
-            updates = [[k, value_to_sql(v)] for k, v in values.items()]
-        else:
-            updates = [[k, v] for k, v in values_sql.items()]
-
-        payload = {"predicate": where, "updates": updates}
-        self._conn._client.post(f"/v1/table/{self.name}/update/", data=payload)
+        self._loop.run_until_complete(
+            self._table.update(where=where, updates=values, updates_sql=values_sql)
+        )

    def cleanup_old_versions(self, *_):
        """cleanup_old_versions() is not supported on the LanceDB cloud"""
@@ -557,11 +479,7 @@ class RemoteTable(Table):
        )

    def count_rows(self, filter: Optional[str] = None) -> int:
-        payload = {"predicate": filter}
-        resp = self._conn._client.post(
-            f"/v1/table/{self.name}/count_rows/", data=payload
-        )
-        return resp
+        return self._loop.run_until_complete(self._table.count_rows(filter))

    def add_columns(self, transforms: Dict[str, str]):
        raise NotImplementedError(
--- a/python/python/lancedb/rerankers/jinaai.py
+++ b/python/python/lancedb/rerankers/jinaai.py
@@ -12,7 +12,6 @@
 #  limitations under the License.

 import os
-import requests
 from functools import cached_property
 from typing import Union

@@ -57,6 +56,8 @@ class JinaReranker(Reranker):

    @cached_property
    def _client(self):
+        import requests
+
        if os.environ.get("JINA_API_KEY") is None and self.api_key is None:
            raise ValueError(
                "JINA_API_KEY not set. Either set it in your environment or \
--- a/python/python/lancedb/rerankers/linear_combination.py
+++ b/python/python/lancedb/rerankers/linear_combination.py
@@ -11,7 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

-from numpy import NaN
+from numpy import nan
 import pyarrow as pa

 from .base import Reranker
@@ -71,7 +71,7 @@ class LinearCombinationReranker(Reranker):
            elif self.score == "all":
                results = results.append_column(
                    "_distance",
-                    pa.array([NaN] * len(fts_results), type=pa.float32()),
+                    pa.array([nan] * len(fts_results), type=pa.float32()),
                )
            return results

@@ -92,7 +92,7 @@ class LinearCombinationReranker(Reranker):
            elif self.score == "all":
                results = results.append_column(
                    "_score",
-                    pa.array([NaN] * len(vector_results), type=pa.float32()),
+                    pa.array([nan] * len(vector_results), type=pa.float32()),
                )
            return results

--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -55,13 +55,14 @@ from .util import (
    safe_import_polars,
    value_to_sql,
 )
+from .index import lang_mapping

 if TYPE_CHECKING:
    import PIL
    from lance.dataset import CleanupStats, ReaderLike
    from ._lancedb import Table as LanceDBTable, OptimizeStats
    from .db import LanceDBConnection
-    from .index import BTree, IndexConfig, IvfPq, Bitmap, LabelList, FTS
+    from .index import BTree, IndexConfig, IvfPq, Bitmap, LabelList, FTS, HnswPq, HnswSq

 pd = safe_import_pandas()
 pl = safe_import_polars()
@@ -497,10 +498,18 @@ class Table(ABC):
        ordering_field_names: Union[str, List[str]] = None,
        *,
        replace: bool = False,
-        with_position: bool = True,
        writer_heap_size: Optional[int] = 1024 * 1024 * 1024,
-        tokenizer_name: str = "default",
        use_tantivy: bool = True,
+        tokenizer_name: Optional[str] = None,
+        with_position: bool = True,
+        # tokenizer configs:
+        base_tokenizer: str = "simple",
+        language: str = "English",
+        max_token_length: Optional[int] = 40,
+        lower_case: bool = True,
+        stem: bool = False,
+        remove_stop_words: bool = False,
+        ascii_folding: bool = False,
    ):
        """Create a full-text search index on the table.

@@ -526,7 +535,6 @@ class Table(ABC):
            The tokenizer to use for the index. Can be "raw", "default" or the 2 letter
            language code followed by "_stem". So for english it would be "en_stem".
            For available languages see: https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html
-            only available with use_tantivy=True for now
        use_tantivy: bool, default True
            If True, use the legacy full-text search implementation based on tantivy.
            If False, use the new full-text search implementation based on lance-index.
@@ -940,7 +948,9 @@ class Table(ABC):
        return _table_uri(self._conn.uri, self.name)

    def _get_fts_index_path(self) -> Tuple[str, pa_fs.FileSystem, bool]:
-        if get_uri_scheme(self._dataset_uri) != "file":
+        from .remote.table import RemoteTable
+
+        if isinstance(self, RemoteTable) or get_uri_scheme(self._dataset_uri) != "file":
            return ("", None, False)
        path = join_uri(self._dataset_uri, "_indices", "fts")
        fs, path = fs_from_uri(path)
@@ -1341,14 +1351,33 @@ class LanceTable(Table):
        ordering_field_names: Union[str, List[str]] = None,
        *,
        replace: bool = False,
-        with_position: bool = True,
        writer_heap_size: Optional[int] = 1024 * 1024 * 1024,
-        tokenizer_name: str = "default",
        use_tantivy: bool = True,
+        tokenizer_name: Optional[str] = None,
+        with_position: bool = True,
+        # tokenizer configs:
+        base_tokenizer: str = "simple",
+        language: str = "English",
+        max_token_length: Optional[int] = 40,
+        lower_case: bool = True,
+        stem: bool = False,
+        remove_stop_words: bool = False,
+        ascii_folding: bool = False,
    ):
        if not use_tantivy:
            if not isinstance(field_names, str):
                raise ValueError("field_names must be a string when use_tantivy=False")
+            tokenizer_configs = {
+                "base_tokenizer": base_tokenizer,
+                "language": language,
+                "max_token_length": max_token_length,
+                "lower_case": lower_case,
+                "stem": stem,
+                "remove_stop_words": remove_stop_words,
+                "ascii_folding": ascii_folding,
+            }
+            if tokenizer_name is not None:
+                tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
            # delete the existing legacy index if it exists
            if replace:
                path, fs, exist = self._get_fts_index_path()
@@ -1359,6 +1388,7 @@ class LanceTable(Table):
                index_type="INVERTED",
                replace=replace,
                with_position=with_position,
+                **tokenizer_configs,
            )
            return

@@ -1381,6 +1411,8 @@ class LanceTable(Table):
                "Full-text search is only supported on the local filesystem"
            )

+        if tokenizer_name is None:
+            tokenizer_name = "default"
        index = create_index(
            path,
            field_names,
@@ -1395,6 +1427,56 @@ class LanceTable(Table):
            writer_heap_size=writer_heap_size,
        )

+    def infer_tokenizer_configs(tokenizer_name: str) -> dict:
+        if tokenizer_name == "default":
+            return {
+                "base_tokenizer": "simple",
+                "language": "English",
+                "max_token_length": 40,
+                "lower_case": True,
+                "stem": False,
+                "remove_stop_words": False,
+                "ascii_folding": False,
+            }
+        elif tokenizer_name == "raw":
+            return {
+                "base_tokenizer": "raw",
+                "language": "English",
+                "max_token_length": None,
+                "lower_case": False,
+                "stem": False,
+                "remove_stop_words": False,
+                "ascii_folding": False,
+            }
+        elif tokenizer_name == "whitespace":
+            return {
+                "base_tokenizer": "whitespace",
+                "language": "English",
+                "max_token_length": None,
+                "lower_case": False,
+                "stem": False,
+                "remove_stop_words": False,
+                "ascii_folding": False,
+            }
+
+        # or it's with language stemming with pattern like "en_stem"
+        if len(tokenizer_name) != 7:
+            raise ValueError(f"Invalid tokenizer name {tokenizer_name}")
+        lang = tokenizer_name[:2]
+        if tokenizer_name[-5:] != "_stem":
+            raise ValueError(f"Invalid tokenizer name {tokenizer_name}")
+        if lang not in lang_mapping:
+            raise ValueError(f"Invalid language code {lang}")
+        return {
+            "base_tokenizer": "simple",
+            "language": lang_mapping[lang],
+            "max_token_length": 40,
+            "lower_case": True,
+            "stem": True,
+            "remove_stop_words": False,
+            "ascii_folding": False,
+        }
+
    def add(
        self,
        data: DATA,
@@ -2302,7 +2384,9 @@ class AsyncTable:
        column: str,
        *,
        replace: Optional[bool] = None,
-        config: Optional[Union[IvfPq, BTree, Bitmap, LabelList, FTS]] = None,
+        config: Optional[
+            Union[IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS]
+        ] = None,
    ):
        """Create an index to speed up queries

@@ -2455,7 +2539,44 @@ class AsyncTable:
    async def _execute_query(
        self, query: Query, batch_size: Optional[int] = None
    ) -> pa.RecordBatchReader:
-        pass
+        # The sync remote table calls into this method, so we need to map the
+        # query to the async version of the query and run that here. This is only
+        # used for that code path right now.
+        async_query = self.query().limit(query.k)
+        if query.offset > 0:
+            async_query = async_query.offset(query.offset)
+        if query.columns:
+            async_query = async_query.select(query.columns)
+        if query.filter:
+            async_query = async_query.where(query.filter)
+        if query.fast_search:
+            async_query = async_query.fast_search()
+        if query.with_row_id:
+            async_query = async_query.with_row_id()
+
+        if query.vector:
+            async_query = (
+                async_query.nearest_to(query.vector)
+                .distance_type(query.metric)
+                .nprobes(query.nprobes)
+            )
+            if query.refine_factor:
+                async_query = async_query.refine_factor(query.refine_factor)
+            if query.vector_column:
+                async_query = async_query.column(query.vector_column)
+
+        if not query.prefilter:
+            async_query = async_query.postfilter()
+
+        if isinstance(query.full_text_query, str):
+            async_query = async_query.nearest_to_text(query.full_text_query)
+        elif isinstance(query.full_text_query, dict):
+            fts_query = query.full_text_query["query"]
+            fts_columns = query.full_text_query.get("columns", []) or []
+            async_query = async_query.nearest_to_text(fts_query, columns=fts_columns)
+
+        table = await async_query.to_arrow()
+        return table.to_reader()

    async def _do_merge(
        self,
@@ -2701,7 +2822,7 @@ class AsyncTable:
            cleanup_older_than = round(cleanup_older_than.total_seconds() * 1000)
        return await self._inner.optimize(cleanup_older_than, delete_unverified)

-    async def list_indices(self) -> IndexConfig:
+    async def list_indices(self) -> Iterable[IndexConfig]:
        """
        List all indices that have been created with Self::create_index
        """
@@ -2785,3 +2906,8 @@ class IndexStatistics:
    ]
    distance_type: Optional[Literal["l2", "cosine", "dot"]] = None
    num_indices: Optional[int] = None
+
+    # This exists for backwards compatibility with an older API, which returned
+    # a dictionary instead of a class.
+    def __getitem__(self, key):
+        return getattr(self, key)
--- a/python/python/tests/test_embeddings.py
+++ b/python/python/tests/test_embeddings.py
@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 from typing import List, Union
+from unittest.mock import MagicMock, patch

 import lance
 import lancedb
@@ -25,6 +26,7 @@ from lancedb.embeddings import (
 )
 from lancedb.embeddings.base import TextEmbeddingFunction
 from lancedb.embeddings.registry import get_registry, register
+from lancedb.embeddings.utils import retry
 from lancedb.pydantic import LanceModel, Vector


@@ -225,3 +227,12 @@ def test_embedding_function_safe_model_dump(embedding_type):
                f"{embedding_type}: Private attribute '{key}' "
                f"is present in dumped model"
            )
+
+
+@patch("time.sleep")
+def test_retry(mock_sleep):
+    test_function = MagicMock(side_effect=[Exception] * 9 + ["result"])
+    test_function = retry()(test_function)
+    result = test_function()
+    assert mock_sleep.call_count == 9
+    assert result == "result"
--- a/python/python/tests/test_embeddings_slow.py
+++ b/python/python/tests/test_embeddings_slow.py
@@ -18,7 +18,6 @@ import lancedb
 import numpy as np
 import pandas as pd
 import pytest
-import requests
 from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector

@@ -108,6 +107,7 @@ def test_basic_text_embeddings(alias, tmp_path):

@pytest.mark.slow
 def test_openclip(tmp_path):
+    import requests
    from PIL import Image

    db = lancedb.connect(tmp_path)
--- a/python/python/tests/test_fts.py
+++ b/python/python/tests/test_fts.py
@@ -235,6 +235,29 @@ async def test_search_fts_async(async_table):
    results = await async_table.query().nearest_to_text("puppy").limit(5).to_list()
    assert len(results) == 5

+    expected_count = await async_table.count_rows(
+        "count > 5000 and contains(text, 'puppy')"
+    )
+    expected_count = min(expected_count, 10)
+
+    limited_results_pre_filter = await (
+        async_table.query()
+        .nearest_to_text("puppy")
+        .where("count > 5000")
+        .limit(10)
+        .to_list()
+    )
+    assert len(limited_results_pre_filter) == expected_count
+    limited_results_post_filter = await (
+        async_table.query()
+        .nearest_to_text("puppy")
+        .where("count > 5000")
+        .limit(10)
+        .postfilter()
+        .to_list()
+    )
+    assert len(limited_results_post_filter) <= expected_count
+

@pytest.mark.asyncio
 async def test_search_fts_specify_column_async(async_table):
--- a/python/python/tests/test_index.py
+++ b/python/python/tests/test_index.py
@@ -49,7 +49,7 @@ async def test_create_scalar_index(some_table: AsyncTable):
    # Can recreate if replace=True
    await some_table.create_index("id", replace=True)
    indices = await some_table.list_indices()
-    assert str(indices) == '[Index(BTree, columns=["id"])]'
+    assert str(indices) == '[Index(BTree, columns=["id"], name="id_idx")]'
    assert len(indices) == 1
    assert indices[0].index_type == "BTree"
    assert indices[0].columns == ["id"]
@@ -64,7 +64,7 @@ async def test_create_scalar_index(some_table: AsyncTable):
 async def test_create_bitmap_index(some_table: AsyncTable):
    await some_table.create_index("id", config=Bitmap())
    indices = await some_table.list_indices()
-    assert str(indices) == '[Index(Bitmap, columns=["id"])]'
+    assert str(indices) == '[Index(Bitmap, columns=["id"], name="id_idx")]'
    indices = await some_table.list_indices()
    assert len(indices) == 1
    index_name = indices[0].name
@@ -80,7 +80,7 @@ async def test_create_bitmap_index(some_table: AsyncTable):
 async def test_create_label_list_index(some_table: AsyncTable):
    await some_table.create_index("tags", config=LabelList())
    indices = await some_table.list_indices()
-    assert str(indices) == '[Index(LabelList, columns=["tags"])]'
+    assert str(indices) == '[Index(LabelList, columns=["tags"], name="tags_idx")]'


@pytest.mark.asyncio
--- a/python/python/tests/test_query.py
+++ b/python/python/tests/test_query.py
@@ -17,6 +17,7 @@ from typing import Optional

 import lance
 import lancedb
+from lancedb.index import IvfPq
 import numpy as np
 import pandas.testing as tm
 import pyarrow as pa
@@ -330,6 +331,12 @@ async def test_query_async(table_async: AsyncTable):
    # Also check an empty query
    await check_query(table_async.query().where("id < 0"), expected_num_rows=0)

+    # with row id
+    await check_query(
+        table_async.query().select(["id", "vector"]).with_row_id(),
+        expected_columns=["id", "vector", "_rowid"],
+    )
+

@pytest.mark.asyncio
 async def test_query_to_arrow_async(table_async: AsyncTable):
@@ -358,6 +365,25 @@ async def test_query_to_pandas_async(table_async: AsyncTable):
    assert df.shape == (0, 4)


+@pytest.mark.asyncio
+async def test_fast_search_async(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    vectors = pa.FixedShapeTensorArray.from_numpy_ndarray(
+        np.random.rand(256, 32)
+    ).storage
+    table = await db.create_table("test", pa.table({"vector": vectors}))
+    await table.create_index(
+        "vector", config=IvfPq(num_partitions=1, num_sub_vectors=1)
+    )
+    await table.add(pa.table({"vector": vectors}))
+
+    q = [1.0] * 32
+    plan = await table.query().nearest_to(q).explain_plan(True)
+    assert "LanceScan" in plan
+    plan = await table.query().nearest_to(q).fast_search().explain_plan(True)
+    assert "LanceScan" not in plan
+
+
 def test_explain_plan(table):
    q = LanceVectorQueryBuilder(table, [0, 0], "vector")
    plan = q.explain_plan(verbose=True)
--- a/python/python/tests/test_remote_client.py
+++ b/python/python/tests/test_remote_client.py
@@ -1,96 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import attrs
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-import pytest
-from aiohttp import web
-from lancedb.remote.client import RestfulLanceDBClient, VectorQuery
-
-
-@attrs.define
-class MockLanceDBServer:
-    runner: web.AppRunner = attrs.field(init=False)
-    site: web.TCPSite = attrs.field(init=False)
-
-    async def query_handler(self, request: web.Request) -> web.Response:
-        table_name = request.match_info["table_name"]
-        assert table_name == "test_table"
-
-        await request.json()
-        # TODO: do some matching
-
-        vecs = pd.Series([np.random.rand(128) for x in range(10)], name="vector")
-        ids = pd.Series(range(10), name="id")
-        df = pd.DataFrame([vecs, ids]).T
-
-        batch = pa.RecordBatch.from_pandas(
-            df,
-            schema=pa.schema(
-                [
-                    pa.field("vector", pa.list_(pa.float32(), 128)),
-                    pa.field("id", pa.int64()),
-                ]
-            ),
-        )
-
-        sink = pa.BufferOutputStream()
-        with pa.ipc.new_file(sink, batch.schema) as writer:
-            writer.write_batch(batch)
-
-        return web.Response(body=sink.getvalue().to_pybytes())
-
-    async def setup(self):
-        app = web.Application()
-        app.add_routes([web.post("/table/{table_name}", self.query_handler)])
-        self.runner = web.AppRunner(app)
-        await self.runner.setup()
-        self.site = web.TCPSite(self.runner, "localhost", 8111)
-
-    async def start(self):
-        await self.site.start()
-
-    async def stop(self):
-        await self.runner.cleanup()
-
-
-@pytest.mark.skip(reason="flaky somehow, fix later")
-@pytest.mark.asyncio
-async def test_e2e_with_mock_server():
-    mock_server = MockLanceDBServer()
-    await mock_server.setup()
-    await mock_server.start()
-
-    try:
-        with RestfulLanceDBClient("lancedb+http://localhost:8111") as client:
-            df = (
-                await client.query(
-                    "test_table",
-                    VectorQuery(
-                        vector=np.random.rand(128).tolist(),
-                        k=10,
-                        _metric="L2",
-                        columns=["id", "vector"],
-                    ),
-                )
-            ).to_pandas()
-
-            assert "vector" in df.columns
-            assert "id" in df.columns
-
-        assert client.closed
-    finally:
-        # make sure we don't leak resources
-        await mock_server.stop()
--- a/python/python/tests/test_remote_db.py
+++ b/python/python/tests/test_remote_db.py
@@ -2,91 +2,19 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

 import contextlib
+from datetime import timedelta
 import http.server
+import json
 import threading
 from unittest.mock import MagicMock
 import uuid

 import lancedb
+from lancedb.conftest import MockTextEmbeddingFunction
+from lancedb.remote import ClientConfig
 from lancedb.remote.errors import HttpError, RetryError
-import pyarrow as pa
-from lancedb.remote.client import VectorQuery, VectorQueryResult
 import pytest
-
-
-class FakeLanceDBClient:
-    def close(self):
-        pass
-
-    def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
-        assert table_name == "test"
-        t = pa.schema([]).empty_table()
-        return VectorQueryResult(t)
-
-    def post(self, path: str):
-        pass
-
-    def mount_retry_adapter_for_table(self, table_name: str):
-        pass
-
-
-def test_remote_db():
-    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
-    setattr(conn, "_client", FakeLanceDBClient())
-
-    table = conn["test"]
-    table.schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
-    table.search([1.0, 2.0]).to_pandas()
-
-
-def test_create_empty_table():
-    client = MagicMock()
-    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
-
-    conn._client = client
-
-    schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
-
-    client.post.return_value = {"status": "ok"}
-    table = conn.create_table("test", schema=schema)
-    assert table.name == "test"
-    assert client.post.call_args[0][0] == "/v1/table/test/create/"
-
-    json_schema = {
-        "fields": [
-            {
-                "name": "vector",
-                "nullable": True,
-                "type": {
-                    "type": "fixed_size_list",
-                    "fields": [
-                        {"name": "item", "nullable": True, "type": {"type": "float"}}
-                    ],
-                    "length": 2,
-                },
-            },
-        ]
-    }
-    client.post.return_value = {"schema": json_schema}
-    assert table.schema == schema
-    assert client.post.call_args[0][0] == "/v1/table/test/describe/"
-
-    client.post.return_value = 0
-    assert table.count_rows(None) == 0
-
-
-def test_create_table_with_recordbatches():
-    client = MagicMock()
-    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
-
-    conn._client = client
-
-    batch = pa.RecordBatch.from_arrays([pa.array([[1.0, 2.0], [3.0, 4.0]])], ["vector"])
-
-    client.post.return_value = {"status": "ok"}
-    table = conn.create_table("test", [batch], schema=batch.schema)
-    assert table.name == "test"
-    assert client.post.call_args[0][0] == "/v1/table/test/create/"
+import pyarrow as pa


 def make_mock_http_handler(handler):
@@ -100,8 +28,35 @@ def make_mock_http_handler(handler):
    return MockLanceDBHandler


+@contextlib.contextmanager
+def mock_lancedb_connection(handler):
+    with http.server.HTTPServer(
+        ("localhost", 8080), make_mock_http_handler(handler)
+    ) as server:
+        handle = threading.Thread(target=server.serve_forever)
+        handle.start()
+
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override="http://localhost:8080",
+            client_config={
+                "retry_config": {"retries": 2},
+                "timeout_config": {
+                    "connect_timeout": 1,
+                },
+            },
+        )
+
+        try:
+            yield db
+        finally:
+            server.shutdown()
+            handle.join()
+
+
@contextlib.asynccontextmanager
-async def mock_lancedb_connection(handler):
+async def mock_lancedb_connection_async(handler):
    with http.server.HTTPServer(
        ("localhost", 8080), make_mock_http_handler(handler)
    ) as server:
@@ -143,7 +98,7 @@ async def test_async_remote_db():
        request.end_headers()
        request.wfile.write(b'{"tables": []}')

-    async with mock_lancedb_connection(handler) as db:
+    async with mock_lancedb_connection_async(handler) as db:
        table_names = await db.table_names()
        assert table_names == []

@@ -159,12 +114,12 @@ async def test_http_error():
        request.end_headers()
        request.wfile.write(b"Internal Server Error")

-    async with mock_lancedb_connection(handler) as db:
-        with pytest.raises(HttpError, match="Internal Server Error") as exc_info:
+    async with mock_lancedb_connection_async(handler) as db:
+        with pytest.raises(HttpError) as exc_info:
            await db.table_names()

        assert exc_info.value.request_id == request_id_holder["request_id"]
-        assert exc_info.value.status_code == 507
+        assert "Internal Server Error" in str(exc_info.value)


@pytest.mark.asyncio
@@ -178,15 +133,225 @@ async def test_retry_error():
        request.end_headers()
        request.wfile.write(b"Try again later")

-    async with mock_lancedb_connection(handler) as db:
-        with pytest.raises(RetryError, match="Hit retry limit") as exc_info:
+    async with mock_lancedb_connection_async(handler) as db:
+        with pytest.raises(RetryError) as exc_info:
            await db.table_names()

        assert exc_info.value.request_id == request_id_holder["request_id"]
-        assert exc_info.value.status_code == 429

        cause = exc_info.value.__cause__
        assert isinstance(cause, HttpError)
        assert "Try again later" in str(cause)
        assert cause.request_id == request_id_holder["request_id"]
        assert cause.status_code == 429
+
+
+@contextlib.contextmanager
+def query_test_table(query_handler):
+    def handler(request):
+        if request.path == "/v1/table/test/describe/":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        elif request.path == "/v1/table/test/query/":
+            content_len = int(request.headers.get("Content-Length"))
+            body = request.rfile.read(content_len)
+            body = json.loads(body)
+
+            data = query_handler(body)
+
+            request.send_response(200)
+            request.send_header("Content-Type", "application/vnd.apache.arrow.file")
+            request.end_headers()
+
+            with pa.ipc.new_file(request.wfile, schema=data.schema) as f:
+                f.write_table(data)
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        assert repr(db) == "RemoteConnect(name=dev)"
+        table = db.open_table("test")
+        assert repr(table) == "RemoteTable(dev.test)"
+        yield table
+
+
+def test_query_sync_minimal():
+    def handler(body):
+        assert body == {
+            "distance_type": "l2",
+            "k": 10,
+            "prefilter": False,
+            "refine_factor": None,
+            "vector": [1.0, 2.0, 3.0],
+            "nprobes": 20,
+        }
+
+        return pa.table({"id": [1, 2, 3]})
+
+    with query_test_table(handler) as table:
+        data = table.search([1, 2, 3]).to_list()
+        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
+        assert data == expected
+
+
+def test_query_sync_maximal():
+    def handler(body):
+        assert body == {
+            "distance_type": "cosine",
+            "k": 42,
+            "prefilter": True,
+            "refine_factor": 10,
+            "vector": [1.0, 2.0, 3.0],
+            "nprobes": 5,
+            "filter": "id > 0",
+            "columns": ["id", "name"],
+            "vector_column": "vector2",
+            "fast_search": True,
+            "with_row_id": True,
+        }
+
+        return pa.table({"id": [1, 2, 3], "name": ["a", "b", "c"]})
+
+    with query_test_table(handler) as table:
+        (
+            table.search([1, 2, 3], vector_column_name="vector2", fast_search=True)
+            .metric("cosine")
+            .limit(42)
+            .refine_factor(10)
+            .nprobes(5)
+            .where("id > 0", prefilter=True)
+            .with_row_id(True)
+            .select(["id", "name"])
+            .to_list()
+        )
+
+
+def test_query_sync_fts():
+    def handler(body):
+        assert body == {
+            "full_text_query": {
+                "query": "puppy",
+                "columns": [],
+            },
+            "k": 10,
+            "vector": [],
+        }
+
+        return pa.table({"id": [1, 2, 3]})
+
+    with query_test_table(handler) as table:
+        (table.search("puppy", query_type="fts").to_list())
+
+    def handler(body):
+        assert body == {
+            "full_text_query": {
+                "query": "puppy",
+                "columns": ["name", "description"],
+            },
+            "k": 42,
+            "vector": [],
+            "with_row_id": True,
+        }
+
+        return pa.table({"id": [1, 2, 3]})
+
+    with query_test_table(handler) as table:
+        (
+            table.search("puppy", query_type="fts", fts_columns=["name", "description"])
+            .with_row_id(True)
+            .limit(42)
+            .to_list()
+        )
+
+
+def test_query_sync_hybrid():
+    def handler(body):
+        if "full_text_query" in body:
+            # FTS query
+            assert body == {
+                "full_text_query": {
+                    "query": "puppy",
+                    "columns": [],
+                },
+                "k": 42,
+                "vector": [],
+                "with_row_id": True,
+            }
+            return pa.table({"_rowid": [1, 2, 3], "_score": [0.1, 0.2, 0.3]})
+        else:
+            # Vector query
+            assert body == {
+                "distance_type": "l2",
+                "k": 42,
+                "prefilter": False,
+                "refine_factor": None,
+                "vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
+                "nprobes": 20,
+                "with_row_id": True,
+            }
+            return pa.table({"_rowid": [1, 2, 3], "_distance": [0.1, 0.2, 0.3]})
+
+    with query_test_table(handler) as table:
+        embedding_func = MockTextEmbeddingFunction()
+        embedding_config = MagicMock()
+        embedding_config.function = embedding_func
+
+        embedding_funcs = MagicMock()
+        embedding_funcs.get = MagicMock(return_value=embedding_config)
+        table.embedding_functions = embedding_funcs
+
+        (table.search("puppy", query_type="hybrid").limit(42).to_list())
+
+
+def test_create_client():
+    mandatory_args = {
+        "uri": "db://dev",
+        "api_key": "fake-api-key",
+        "region": "us-east-1",
+    }
+
+    db = lancedb.connect(**mandatory_args)
+    assert isinstance(db.client_config, ClientConfig)
+
+    db = lancedb.connect(**mandatory_args, client_config={})
+    assert isinstance(db.client_config, ClientConfig)
+
+    db = lancedb.connect(
+        **mandatory_args,
+        client_config=ClientConfig(timeout_config={"connect_timeout": 42}),
+    )
+    assert isinstance(db.client_config, ClientConfig)
+    assert db.client_config.timeout_config.connect_timeout == timedelta(seconds=42)
+
+    db = lancedb.connect(
+        **mandatory_args,
+        client_config={"timeout_config": {"connect_timeout": timedelta(seconds=42)}},
+    )
+    assert isinstance(db.client_config, ClientConfig)
+    assert db.client_config.timeout_config.connect_timeout == timedelta(seconds=42)
+
+    db = lancedb.connect(
+        **mandatory_args, client_config=ClientConfig(retry_config={"retries": 42})
+    )
+    assert isinstance(db.client_config, ClientConfig)
+    assert db.client_config.retry_config.retries == 42
+
+    db = lancedb.connect(
+        **mandatory_args, client_config={"retry_config": {"retries": 42}}
+    )
+    assert isinstance(db.client_config, ClientConfig)
+    assert db.client_config.retry_config.retries == 42
+
+    with pytest.warns(DeprecationWarning):
+        db = lancedb.connect(**mandatory_args, connection_timeout=42)
+        assert db.client_config.timeout_config.connect_timeout == timedelta(seconds=42)
+
+    with pytest.warns(DeprecationWarning):
+        db = lancedb.connect(**mandatory_args, read_timeout=42)
+        assert db.client_config.timeout_config.read_timeout == timedelta(seconds=42)
+
+    with pytest.warns(DeprecationWarning):
+        lancedb.connect(**mandatory_args, request_thread_pool=10)
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -991,13 +991,10 @@ def test_count_rows(db):
    assert table.count_rows(filter="text='bar'") == 1


-def test_hybrid_search(db, tmp_path):
-    # This test uses an FTS index
-    pytest.importorskip("lancedb.fts")
-
+def setup_hybrid_search_table(tmp_path, embedding_func):
    db = MockDB(str(tmp_path))
    # Create a LanceDB table schema with a vector and a text column
-    emb = EmbeddingFunctionRegistry.get_instance().get("test")()
+    emb = EmbeddingFunctionRegistry.get_instance().get(embedding_func)()

    class MyTable(LanceModel):
        text: str = emb.SourceField()
@@ -1030,6 +1027,15 @@ def test_hybrid_search(db, tmp_path):
    # Create a fts index
    table.create_fts_index("text")

+    return table, MyTable, emb
+
+
+def test_hybrid_search(tmp_path):
+    # This test uses an FTS index
+    pytest.importorskip("lancedb.fts")
+
+    table, MyTable, emb = setup_hybrid_search_table(tmp_path, "test")
+
    result1 = (
        table.search("Our father who art in heaven", query_type="hybrid")
        .rerank(normalize="score")
@@ -1094,6 +1100,24 @@ def test_hybrid_search(db, tmp_path):
        table.search(query_type="hybrid").text("Arrrrggghhhhhhh").to_list()


+def test_hybrid_search_metric_type(db, tmp_path):
+    # This test uses an FTS index
+    pytest.importorskip("lancedb.fts")
+
+    # Need to use nonnorm as the embedding function so L2 and dot results
+    # are different
+    table, _, _ = setup_hybrid_search_table(tmp_path, "nonnorm")
+
+    # with custom metric
+    result_dot = (
+        table.search("feeling lucky", query_type="hybrid").metric("dot").to_arrow()
+    )
+    result_l2 = table.search("feeling lucky", query_type="hybrid").to_arrow()
+    assert len(result_dot) > 0
+    assert len(result_l2) > 0
+    assert result_dot["_relevance_score"] != result_l2["_relevance_score"]
+
+
@pytest.mark.parametrize(
    "consistency_interval", [None, timedelta(seconds=0), timedelta(seconds=0.1)]
 )
--- a/python/src/connection.rs
+++ b/python/src/connection.rs
@@ -170,6 +170,17 @@ impl Connection {
        })
    }

+    pub fn rename_table(
+        self_: PyRef<'_, Self>,
+        old_name: String,
+        new_name: String,
+    ) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.get_inner()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.rename_table(old_name, new_name).await.infer_error()
+        })
+    }
+
    pub fn drop_table(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.get_inner()?.clone();
        future_into_py(self_.py(), async move {
--- a/python/src/index.rs
+++ b/python/src/index.rs
@@ -24,8 +24,8 @@ use lancedb::{
    DistanceType,
 };
 use pyo3::{
-    exceptions::{PyRuntimeError, PyValueError},
-    pyclass, pymethods, PyResult,
+    exceptions::{PyKeyError, PyRuntimeError, PyValueError},
+    pyclass, pymethods, IntoPy, PyObject, PyResult, Python,
 };

 use crate::util::parse_distance_type;
@@ -106,12 +106,41 @@ impl Index {
        })
    }

+    #[allow(clippy::too_many_arguments)]
    #[staticmethod]
-    pub fn fts(with_position: Option<bool>) -> Self {
+    pub fn fts(
+        with_position: Option<bool>,
+        base_tokenizer: Option<String>,
+        language: Option<String>,
+        max_token_length: Option<usize>,
+        lower_case: Option<bool>,
+        stem: Option<bool>,
+        remove_stop_words: Option<bool>,
+        ascii_folding: Option<bool>,
+    ) -> Self {
        let mut opts = FtsIndexBuilder::default();
        if let Some(with_position) = with_position {
            opts = opts.with_position(with_position);
        }
+        if let Some(base_tokenizer) = base_tokenizer {
+            opts.tokenizer_configs = opts.tokenizer_configs.base_tokenizer(base_tokenizer);
+        }
+        if let Some(language) = language {
+            opts.tokenizer_configs = opts.tokenizer_configs.language(&language).unwrap();
+        }
+        opts.tokenizer_configs = opts.tokenizer_configs.max_token_length(max_token_length);
+        if let Some(lower_case) = lower_case {
+            opts.tokenizer_configs = opts.tokenizer_configs.lower_case(lower_case);
+        }
+        if let Some(stem) = stem {
+            opts.tokenizer_configs = opts.tokenizer_configs.stem(stem);
+        }
+        if let Some(remove_stop_words) = remove_stop_words {
+            opts.tokenizer_configs = opts.tokenizer_configs.remove_stop_words(remove_stop_words);
+        }
+        if let Some(ascii_folding) = ascii_folding {
+            opts.tokenizer_configs = opts.tokenizer_configs.ascii_folding(ascii_folding);
+        }
        Self {
            inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
        }
@@ -207,7 +236,21 @@ pub struct IndexConfig {
 #[pymethods]
 impl IndexConfig {
    pub fn __repr__(&self) -> String {
-        format!("Index({}, columns={:?})", self.index_type, self.columns)
+        format!(
+            "Index({}, columns={:?}, name=\"{}\")",
+            self.index_type, self.columns, self.name
+        )
+    }
+
+    // For backwards-compatibility with the old sync SDK, we also support getting
+    // attributes via __getitem__.
+    pub fn __getitem__(&self, key: String, py: Python<'_>) -> PyResult<PyObject> {
+        match key.as_str() {
+            "index_type" => Ok(self.index_type.clone().into_py(py)),
+            "columns" => Ok(self.columns.clone().into_py(py)),
+            "name" | "index_name" => Ok(self.name.clone().into_py(py)),
+            _ => Err(PyKeyError::new_err(format!("Invalid key: {}", key))),
+        }
    }
 }

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Lance Release	dd9ce337e2	Bump version: 0.13.0-beta.0 → 0.13.0-beta.1	2024-11-06 17:26:17 +00:00
Will Jones	b9921d56cc	fix(node): update default log level to warn (#1801 ) 🤦	2024-11-06 09:13:53 -08:00
Lance Release	0cfd9ed18e	Updating package-lock.json	2024-11-05 23:21:50 +00:00
Lance Release	975398c3a8	Bump version: 0.12.0 → 0.13.0-beta.0	2024-11-05 23:21:32 +00:00
Lance Release	08d5f93f34	Bump version: 0.15.0 → 0.16.0-beta.0	2024-11-05 23:21:13 +00:00
Will Jones	91cab3b556	feat(python): transition Python remote sdk to use Rust implementation (#1701 ) * Replaces Python implementation of Remote SDK with Rust one. * Drops dependency on `attrs` and `cachetools`. Makes `requests` an optional dependency used only for embeddings feature. * Adds dependency on `nest-asyncio`. This was required to get hybrid search working. * Deprecate `request_thread_pool` parameter. We now use the tokio threadpool. * Stop caching the `schema` on a remote table. Schema is mutable and there's no mechanism in place to invalidate the cache. * Removed the client-side resolution of the vector column. We should already be resolving this server-side.	2024-11-05 13:44:39 -08:00
Will Jones	c61bfc3af8	chore: update package locks (#1798 )	2024-11-05 13:28:59 -08:00
Bert	4e8c7b0adf	fix: serialize vectordb client errors as json (#1795 )	2024-11-05 14:16:25 -05:00
Weston Pace	26f4a80e10	feat: upgrade to lance 0.19.2-beta.3 (#1794 )	2024-11-05 06:43:41 -08:00
Will Jones	3604d20ad3	feat(python,node): support with_row_id in Python and remote (#1784 ) Needed to support hybrid search in Remote SDK.	2024-11-04 11:25:45 -08:00
Gagan Bhullar	9708d829a9	fix: explain plan options (#1776 ) PR fixes #1768	2024-11-04 10:25:34 -08:00
Will Jones	059c9794b5	fix(rust): fix update, open_table, fts search in remote client (#1785 ) * `open_table` uses `POST` not `GET` * `update` uses `predicate` key not `only_if` * For FTS search, vector cannot be omitted. It must be passed as empty. * Added logging of JSON request bodies to debug level logging.	2024-11-04 08:27:55 -08:00
Will Jones	15ed7f75a0	feat(python): support post filter on FTS (#1783 )	2024-11-01 10:05:05 -07:00
Will Jones	96181ab421	feat: `fast_search` in Python and Node (#1623 ) Sometimes it is acceptable to users to only search indexed data and skip and new un-indexed data. For example, if un-indexed data will be shortly indexed and they don't mind the delay. In these cases, we can save a lot of CPU time in search, and provide better latency. Users can activate this on queries using `fast_search()`.	2024-11-01 09:29:09 -07:00
Will Jones	f3fc339ef6	fix(rust): fix delete, update, query in remote SDK (#1782 ) Fixes several minor issues with Rust remote SDK: * Delete uses `predicate` not `filter` as parameter * Update does not return the row value in remote SDK * Update takes tuples * Content type returned by query node is wrong, so we shouldn't validate it. https://github.com/lancedb/sophon/issues/2742 * Data returned by query endpoint is actually an Arrow IPC file, not IPC stream.	2024-10-31 15:22:09 -07:00
Will Jones	113cd6995b	fix: index_stats works for FTS indices (#1780 ) When running `index_stats()` for an FTS index, users would get the deserialization error: ``` InvalidInput { message: "error deserializing index statistics: unknown variant `Inverted`, expected one of `IvfPq`, `IvfHnswPq`, `IvfHnswSq`, `BTree`, `Bitmap`, `LabelList`, `FTS` at line 1 column 24" } ```	2024-10-30 11:33:49 -07:00
Lance Release	02535bdc88	Updating package-lock.json	2024-10-29 22:16:51 +00:00
Lance Release	facc7d61c0	Bump version: 0.12.0-beta.0 → 0.12.0	2024-10-29 22:16:32 +00:00
Lance Release	f947259f16	Bump version: 0.11.1-beta.1 → 0.12.0-beta.0	2024-10-29 22:16:27 +00:00
Lance Release	e291212ecf	Bump version: 0.15.0-beta.0 → 0.15.0	2024-10-29 22:16:05 +00:00
Lance Release	edc6445f6f	Bump version: 0.14.1-beta.1 → 0.15.0-beta.0	2024-10-29 22:16:05 +00:00
Will Jones	a324f4ad7a	feat(node): enable logging and show full errors (#1775 ) This exposes the `LANCEDB_LOG` environment variable in node, so that users can now turn on logging. In addition, fixes a bug where only the top-level error from Rust was being shown. This PR makes sure the full error chain is included in the error message. In the future, will improve this so the error chain is set on the [cause](https://nodejs.org/api/errors.html#errorcause) property of JS errors https://github.com/lancedb/lancedb/issues/1779 Fixes #1774	2024-10-29 15:13:34 -07:00
Weston Pace	55104c5bae	feat: allow distance type (metric) to be specified during hybrid search (#1777 )	2024-10-29 13:51:18 -07:00
Rithik Kumar	d71df4572e	docs: revamp langchain integration page (#1773 ) Before - <img width="1030" alt="Screenshot 2024-10-28 132932" src="https://github.com/user-attachments/assets/63f78bfa-949e-473e-ab22-0c692577fa3e"> After - <img width="1037" alt="Screenshot 2024-10-28 132727" src="https://github.com/user-attachments/assets/85a12f6c-74f0-49ba-9f1a-fe77ad125704">	2024-10-29 22:55:50 +05:30
Rithik Kumar	aa269199ad	docs: fix archived examples links (#1751 )	2024-10-29 22:55:27 +05:30
BubbleCal	32fdcf97db	feat!: upgrade lance to 0.19.1 (#1762 ) BREAKING CHANGE: default tokenizer no longer does stemming or stop-word removal. Users should explicitly turn that option on in the future. - upgrade lance to 0.19.1 - update the FTS docs - update the FTS API Upstream change notes: https://github.com/lancedb/lance/releases/tag/v0.19.1 --------- Signed-off-by: BubbleCal <bubble-cal@outlook.com> Co-authored-by: Will Jones <willjones127@gmail.com>	2024-10-29 09:03:52 -07:00
Ryan Green	b9802a0d23	Revert "fix: error during deserialization of "INVERTED" index type" This reverts commit `2ea5939f85`.	2024-10-25 14:46:47 -02:30
Ryan Green	2ea5939f85	fix: error during deserialization of "INVERTED" index type	2024-10-25 14:40:14 -02:30
Lance Release	04e1f1ee4c	Updating package-lock.json	2024-10-23 00:34:22 +00:00
Lance Release	bbc588e27d	Bump version: 0.11.1-beta.0 → 0.11.1-beta.1	2024-10-23 00:34:01 +00:00
Lance Release	5517e102c3	Bump version: 0.14.1-beta.0 → 0.14.1-beta.1	2024-10-23 00:33:40 +00:00
Will Jones	82197c54e4	perf: eliminate iop in refresh (#1760 ) Closes #1741 If we checkout a version, we need to make a `HEAD` request to get the size of the manifest. The new `checkout_latest()` code path can skip this IOP. This makes the refresh slightly faster.	2024-10-18 13:40:24 -07:00
Will Jones	48f46d4751	docs(node): update `indexStats` signature and regenerate docs (#1742 ) `indexStats` still referenced UUID even though in https://github.com/lancedb/lancedb/pull/1702 we changed it to take name instead.	2024-10-18 10:53:28 -07:00
Lance Release	437316cbbc	Updating package-lock.json	2024-10-17 18:59:18 +00:00
Lance Release	d406eab2c8	Bump version: 0.11.0 → 0.11.1-beta.0	2024-10-17 18:59:01 +00:00
Lance Release	1f41101897	Bump version: 0.14.0 → 0.14.1-beta.0	2024-10-17 18:58:45 +00:00
Will Jones	99e4db0d6a	feat(rust): allow `add_embedding` on `create_empty_table` (#1754 ) Fixes https://github.com/lancedb/lancedb/issues/1750	2024-10-17 11:58:15 -07:00
Will Jones	46486d4d22	fix: list_indices can handle fts indexes (#1753 ) Fixes #1752	2024-10-16 10:39:40 -07:00
Weston Pace	f43cb8bba1	feat: upgrade lance to 0.18.3 (#1748 )	2024-10-16 00:48:31 -07:00
James Wu	38eb05f297	fix(python): remove dependency on retry package (#1749 ) ## user story fixes https://github.com/lancedb/lancedb/issues/1480 https://github.com/invl/retry has not had an update in 8 years, one if its sub-dependencies via requirements.txt (https://github.com/pytest-dev/py) is no longer maintained and has a high severity vulnerability (CVE-2022-42969). retry is only used for a single function in the python codebase for a deprecated helper function `with_embeddings`, which was created for an older tutorial (https://github.com/lancedb/lancedb/pull/12) [but is now deprecated](https://lancedb.github.io/lancedb/embeddings/legacy/). ## changes i backported a limited range of functionality of the `@retry()` decorator directly into lancedb so that we no longer have a dependency to the `retry` package. ## tests ``` /Users/james/src/lancedb/python $ ruff check . All checks passed! /Users/james/src/lancedb/python $ pytest python/tests/test_embeddings.py python/tests/test_embeddings.py .......s.... [100%] ================================================================ 11 passed, 1 skipped, 2 warnings in 7.08s ================================================================ ```	2024-10-15 15:13:57 -07:00
Ryan Green	679a70231e	feat: allow fast_search on python remote table (#1747 ) Add `fast_search` parameter to query builder and remote table to support skipping flat search in remote search	2024-10-14 14:39:54 -06:00
Dominik Weckmüller	e7b56b7b2a	docs: add permanent link chain icon to headings without impacting SEO (#1746 ) I noted that there are no permanent links in the docs. Adapted the current best solution from https://github.com/squidfunk/mkdocs-material/discussions/3535. It adds a GitHub-like chain icon to the left of each heading (right on mobile) and does not impact SEO unlike the default solution with pilcrow char `¶` that might show up on google search results. <img alt="image" src="https://user-images.githubusercontent.com/182589/153004627-6df3f8e9-c747-4f43-bd62-a8dabaa96c3f.gif">	2024-10-14 11:58:23 -07:00
Olzhas Alexandrov	5ccd0edec2	docs: clarify infrastructure requirements for S3 Express One Zone (#1745 )	2024-10-11 14:06:28 -06:00
Will Jones	9c74c435e0	ci: update package lock (#1740 )	2024-10-09 15:14:08 -06:00
Lance Release	6de53ce393	Updating package-lock.json	2024-10-09 18:54:29 +00:00
Lance Release	9f42fbba96	Bump version: 0.11.0-beta.2 → 0.11.0	2024-10-09 18:54:09 +00:00
Lance Release	d892f7a622	Bump version: 0.11.0-beta.1 → 0.11.0-beta.2	2024-10-09 18:54:04 +00:00