Bump version: 0.14.0-beta.1 → 0.14.0

Includes changes from v0.18.1 and v0.18.2:

* [v0.18.1 change
log](https://github.com/lancedb/lance/releases/tag/v0.18.1)
* [v0.18.2 change
log](https://github.com/lancedb/lance/releases/tag/v0.18.2)

Closes #1656
Closes #1615
Closes #1661

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.10.0-beta.1"
+current_version = "0.11.0-beta.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.
@@ -24,34 +24,87 @@ commit = true
 message = "Bump version: {current_version} → {new_version}"
 commit_args = ""
 
+# Java maven files
+pre_commit_hooks = [
+  """
+    NEW_VERSION="${BVHOOK_NEW_MAJOR}.${BVHOOK_NEW_MINOR}.${BVHOOK_NEW_PATCH}"
+    if [ ! -z "$BVHOOK_NEW_PRE_L" ] && [ ! -z "$BVHOOK_NEW_PRE_N" ]; then
+        NEW_VERSION="${NEW_VERSION}-${BVHOOK_NEW_PRE_L}.${BVHOOK_NEW_PRE_N}"
+    fi
+    echo "Constructed new version: $NEW_VERSION"
+    cd java && mvn versions:set -DnewVersion=$NEW_VERSION && mvn versions:commit
+
+    # Check for any modified but unstaged pom.xml files
+    MODIFIED_POMS=$(git ls-files -m | grep pom.xml)
+    if [ ! -z "$MODIFIED_POMS" ]; then
+        echo "The following pom.xml files were modified but not staged. Adding them now:"
+        echo "$MODIFIED_POMS" | while read -r file; do
+            git add "$file"
+            echo "Added: $file"
+        done
+    fi
+    """,
+]
+
 [tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
 optional_value = "final"
+values = ["beta", "final"]
 
 [[tool.bumpversion.files]]
 filename = "node/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 [[tool.bumpversion.files]]
 filename = "nodejs/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
 
 # nodejs binary packages
 [[tool.bumpversion.files]]
 glob = "nodejs/npm/*/package.json"
-search = "\"version\": \"{current_version}\","
 replace = "\"version\": \"{new_version}\","
+search = "\"version\": \"{current_version}\","
+
+# vectodb node binary packages
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-darwin-arm64\": \"{new_version}\""
+search = "\"@lancedb/vectordb-darwin-arm64\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-darwin-x64\": \"{new_version}\""
+search = "\"@lancedb/vectordb-darwin-x64\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-arm64-gnu\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\""
+search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\""
+
+[[tool.bumpversion.files]]
+glob = "node/package.json"
+replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\""
+search = "\"@lancedb/vectordb-win32-x64-msvc\": \"{current_version}\""
 
 # Cargo files
 # ------------
 [[tool.bumpversion.files]]
 filename = "rust/ffi/node/Cargo.toml"
-search = "\nversion = \"{current_version}\""
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
 
 [[tool.bumpversion.files]]
 filename = "rust/lancedb/Cargo.toml"
-search = "\nversion = \"{current_version}\""
 replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""
+
+[[tool.bumpversion.files]]
+filename = "nodejs/Cargo.toml"
+replace = "\nversion = \"{new_version}\""
+search = "\nversion = \"{current_version}\""

.github/workflows/docs_test.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: ubuntu-24.04
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -60,7 +60,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "warp-ubuntu-latest-x64-4x"
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/java-publish.yml

@@ -94,11 +94,16 @@ jobs:
           mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64
           cp ../liblancedb_jni_darwin_aarch64.zip/liblancedb_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblancedb_jni.dylib
           cp ../liblancedb_jni_linux_aarch64.zip/liblancedb_jni.so ./core/target/classes/nativelib/linux-aarch64/liblancedb_jni.so
+      - name: Dry run
+        if: github.event_name == 'pull_request'
+        run: |
+          mvn --batch-mode -DskipTests package
       - name: Set github
         run: |
           git config --global user.email "LanceDB Github Runner"
           git config --global user.name "dev+gha@lancedb.com"
       - name: Publish with Java 8
+        if: github.event_name == 'release'
         run: |
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf

.github/workflows/make-release-commit.yml

@@ -30,7 +30,7 @@ on:
         default: true
         type: boolean
       other:
-        description: 'Make a Node/Rust release'
+        description: 'Make a Node/Rust/Java release'
         required: true
         default: true
         type: boolean

.github/workflows/rust.yml

@@ -26,15 +26,14 @@ env:
 jobs:
   lint:
     timeout-minutes: 30
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     defaults:
       run:
         shell: bash
-        working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
+      CC: clang-18
-      CXX: g++-12
+      CXX: clang++-18
     steps:
     - uses: actions/checkout@v4
       with:
@@ -50,21 +49,21 @@ jobs:
     - name: Run format
       run: cargo fmt --all -- --check
     - name: Run clippy
-      run: cargo clippy --all --all-features -- -D warnings
+      run: cargo clippy --workspace --tests --all-features -- -D warnings
   linux:
     timeout-minutes: 30
     # To build all features, we need more disk space than is available
-    # on the GitHub-provided runner. This is mostly due to the the
+    # on the free OSS github runner. This is mostly due to the the
     # sentence-transformers feature.
-    runs-on: warp-ubuntu-latest-x64-4x
+    runs-on: ubuntu-2404-4x-x64
     defaults:
       run:
         shell: bash
         working-directory: rust
     env:
       # Need up-to-date compilers for kernels
-      CC: gcc-12
+      CC: clang-18
-      CXX: g++-12
+      CXX: clang++-18
     steps:
     - uses: actions/checkout@v4
       with:
@@ -77,6 +76,12 @@ jobs:
       run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
+    - name: Make Swap
+      run: |
+        sudo fallocate -l 16G /swapfile
+        sudo chmod 600 /swapfile
+        sudo mkswap /swapfile
+        sudo swapon /swapfile
     - name: Start S3 integration test environment
       working-directory: .
       run: docker compose up --detach --wait

Cargo.toml

@@ -20,13 +20,13 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.17.0", "features" = ["dynamodb"] }
+lance = { "version" = "=0.18.2", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.17.0" }
+lance-index = { "version" = "=0.18.2" }
-lance-linalg = { "version" = "=0.17.0" }
+lance-linalg = { "version" = "=0.18.2" }
-lance-table = { "version" = "=0.17.0" }
+lance-table = { "version" = "=0.18.2" }
-lance-testing = { "version" = "=0.17.0" }
+lance-testing = { "version" = "=0.18.2" }
-lance-datafusion = { "version" = "=0.17.0" }
+lance-datafusion = { "version" = "=0.18.2" }
-lance-encoding = { "version" = "=0.17.0" }
+lance-encoding = { "version" = "=0.18.2" }
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"
@@ -38,16 +38,19 @@ arrow-arith = "52.2"
 arrow-cast = "52.2"
 async-trait = "0"
 chrono = "0.4.35"
-datafusion-physical-plan = "40.0"
+datafusion-common = "41.0"
+datafusion-physical-plan = "41.0"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
+moka = { version = "0.11", features = ["future"] }
 object_store = "0.10.2"
 pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
+rand = "0.8"
 regex = "1.10"
 lazy_static = "1"

README.md

@@ -82,4 +82,4 @@ result = table.search([100, 100]).limit(2).to_pandas()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/vectordb-recipes/tree/main/examples/Youtube-Search-QA-Bot">Build a question and answer bot with LanceDB</a>

docs/mkdocs.yml

@@ -34,6 +34,7 @@ theme:
     - navigation.footer
     - navigation.tracking
     - navigation.instant
+    - content.footnote.tooltips
   icon:
     repo: fontawesome/brands/github
     annotation: material/arrow-right-circle
@@ -65,6 +66,11 @@ plugins:
 markdown_extensions:
   - admonition
   - footnotes
+  - pymdownx.critic
+  - pymdownx.caret
+  - pymdownx.keys
+  - pymdownx.mark
+  - pymdownx.tilde
   - pymdownx.details
   - pymdownx.highlight:
       anchor_linenums: true
@@ -106,6 +112,18 @@ nav:
               - Overview: hybrid_search/hybrid_search.md
               - Comparing Rerankers: hybrid_search/eval.md
               - Airbnb financial data example: notebooks/hybrid_search.ipynb
+          - RAG:
+              - Vanilla RAG: rag/vanilla_rag.md
+              - Multi-head RAG: rag/multi_head_rag.md
+              - Corrective RAG: rag/corrective_rag.md
+              - Agentic RAG: rag/agentic_rag.md
+              - Graph RAG: rag/graph_rag.md
+              - Self RAG: rag/self_rag.md
+              - Adaptive RAG: rag/adaptive_rag.md
+              - SFR RAG: rag/sfr_rag.md
+              - Advanced Techniques:
+                - HyDE: rag/advanced_techniques/hyde.md
+                - FLARE: rag/advanced_techniques/flare.md
           - Reranking:
               - Quickstart: reranking/index.md
               - Cohere Reranker: reranking/cohere.md
@@ -127,7 +145,8 @@ nav:
               - Reranking: guides/tuning_retrievers/2_reranking.md
               - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
-          - Overview: embeddings/index.md
+          - Understand Embeddings: embeddings/understanding_embeddings.md 
+          - Get Started: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
           - Available models: 
               - Overview: embeddings/default_embedding_functions.md
@@ -165,6 +184,7 @@ nav:
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
           - dlt: integrations/dlt.md
+          - phidata: integrations/phidata.md
       - 🎯 Examples:
           - Overview: examples/index.md
           - 🐍 Python:
@@ -220,6 +240,18 @@ nav:
           - Overview: hybrid_search/hybrid_search.md
           - Comparing Rerankers: hybrid_search/eval.md
           - Airbnb financial data example: notebooks/hybrid_search.ipynb
+      - RAG:
+          - Vanilla RAG: rag/vanilla_rag.md
+          - Multi-head RAG: rag/multi_head_rag.md
+          - Corrective RAG: rag/corrective_rag.md
+          - Agentic RAG: rag/agentic_rag.md
+          - Graph RAG: rag/graph_rag.md
+          - Self RAG: rag/self_rag.md
+          - Adaptive RAG: rag/adaptive_rag.md
+          - SFR RAG: rag/sfr_rag.md
+          - Advanced Techniques:
+            - HyDE: rag/advanced_techniques/hyde.md
+            - FLARE: rag/advanced_techniques/flare.md
       - Reranking:
           - Quickstart: reranking/index.md
           - Cohere Reranker: reranking/cohere.md
@@ -241,7 +273,8 @@ nav:
           - Reranking: guides/tuning_retrievers/2_reranking.md
           - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
-      - Overview: embeddings/index.md
+      - Understand Embeddings: embeddings/understanding_embeddings.md 
+      - Get Started: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
       - Available models: 
           - Overview: embeddings/default_embedding_functions.md
@@ -275,6 +308,7 @@ nav:
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
       - dlt: integrations/dlt.md
+      - phidata: integrations/phidata.md
   - Examples:
       - examples/index.md
       - 🐍 Python:
@@ -330,4 +364,5 @@ extra:
     - icon: fontawesome/brands/x-twitter
       link: https://twitter.com/lancedb
     - icon: fontawesome/brands/linkedin
-      link: https://www.linkedin.com/company/lancedb
+      link: https://www.linkedin.com/company/lancedb      
+      

docs/src/embeddings/available_embedding_models/text_embedding_functions/huggingface_embedding.md

@@ -1,5 +1,5 @@
 # Huggingface embedding models
-We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
+We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. 
 
 Example usage - 
 ```python

docs/src/embeddings/understanding_embeddings.md

@@ -0,0 +1,133 @@
+# Understand Embeddings
+
+The term **dimension** is a synonym for the number of elements in a feature vector. Each feature can be thought of as a different axis in a geometric space. 
+
+High-dimensional data means there are many features(or attributes) in the data.
+
+!!! example
+     1. An image is a data point and it might have thousands of dimensions because each pixel could be considered as a feature. 
+
+     2. Text data, when represented by each word or character, can also lead to high dimensions, especially when considering all possible words in a language.
+
+Embedding captures **meaning and relationships** within data by mapping high-dimensional data into a lower-dimensional space. It captures it by placing inputs that are more **similar in meaning** closer together in the **embedding space**. 
+
+## What are Vector Embeddings?
+
+Vector embeddings is a way to convert complex data, like text, images, or audio into numerical coordinates (called vectors) that can be plotted in an n-dimensional space(embedding space). 
+
+The closer these data points are related in the real world, the closer their corresponding numerical coordinates (vectors) will be to each other in the embedding space. This proximity in the embedding space reflects their semantic similarities, allowing machines to intuitively understand and process the data in a way that mirrors human perception of relationships and meaning.
+
+In a way, it captures the most important aspects of the data while ignoring the less important ones. As a result, tasks like searching for related content or identifying patterns become more efficient and accurate, as the embeddings make it possible to quantify how **closely related** different **data points** are and **reduce** the **computational complexity**.
+
+??? question "Are vectors and embeddings the same thing?"
+
+    When we say “vectors” we mean - **list of numbers** that **represents the data**. 
+    When we say “embeddings” we mean - **list of numbers** that **capture important details and relationships**.
+
+    Although the terms are often used interchangeably, “embeddings” highlight how the data is represented with meaning and structure, while “vector” simply refers to the numerical form of that representation.
+
+## Embedding vs Indexing
+
+We already saw that creating **embeddings** on data is a method of creating **vectors** for a **n-dimensional embedding space** that captures the meaning and relationships inherent in the data.
+
+Once we have these **vectors**, indexing comes into play. Indexing is a method of organizing these vector embeddings, that allows us to quickly and efficiently locate and retrieve them from the entire dataset of vector embeddings.
+
+## What types of data/objects can be embedded?
+
+The following are common types of data that can be embedded:
+
+1. **Text**: Text data includes sentences, paragraphs, documents, or any written content.
+2. **Images**:  Image data encompasses photographs, illustrations, or any visual content.
+3. **Audio**: Audio data includes sounds, music, speech, or any auditory content.
+4. **Video**:  Video data consists of moving images and sound, which can convey complex information.
+
+Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model.
+
+!!! tip "LanceDB vs Other traditional Vector DBs"
+    While many vector databases primarily focus on the storage and retrieval of vector embeddings, **LanceDB** uses **Lance file format** (operates on a disk-based architecture), which allows for the storage and management of not just embeddings but also **raw file data (bytes)**. This capability means that users can integrate various types of data, including images and text, alongside their vector embeddings in a unified system.
+
+    With the ability to store both vectors and associated file data, LanceDB enhances the querying process. Users can perform semantic searches that not only retrieve similar embeddings but also access related files and metadata, thus streamlining the workflow.
+
+## How does embedding works?
+
+As mentioned, after creating embedding, each data point is represented as a vector in a n-dimensional space (embedding space). The dimensionality of this space can vary depending on the complexity of the data and the specific embedding technique used.
+
+Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar. To quantify this closeness, we use distance as a metric which can be measured in the  following way - 
+
+1. **Euclidean Distance (L2)**: It calculates the straight-line distance between two points (vectors) in a multidimensional space.
+2. **Cosine Similarity**: It measures the cosine of the angle between two vectors, providing a normalized measure of similarity based on their direction.
+3. **Dot product**: It is calculated as the sum of the products of their corresponding components. To measure relatedness it considers both the magnitude and direction of the vectors.
+
+## How do you create and store vector embeddings for your data?
+
+1. **Creating embeddings**: Choose an embedding model, it can be a pre-trained model (open-source or commercial) or you can train a custom embedding model for your scenario. Then feed your preprocessed data into the chosen model to obtain embeddings.
+
+??? question "Popular choices for embedding models"
+    For text data, popular choices are OpenAI’s text-embedding models, Google Gemini text-embedding models, Cohere’s Embed models, and SentenceTransformers, etc.
+
+    For image data, popular choices are CLIP (Contrastive Language–Image Pretraining), Imagebind embeddings by meta (supports audio, video, and image), and Jina multi-modal embeddings, etc.
+
+2. **Storing vector embeddings**: This effectively requires **specialized databases** that can handle the complexity of vector data, as traditional databases often struggle with this task. Vector databases are designed specifically for storing and querying vector embeddings. They optimize for efficient nearest-neighbor searches and provide built-in indexing mechanisms.
+
+!!! tip "Why LanceDB"
+    LanceDB **automates** the entire process of creating and storing embeddings for your data. LanceDB allows you to define and use **embedding functions**, which can be **pre-trained models** or **custom models**. 
+    
+    This enables you to **generate** embeddings tailored to the nature of your data (e.g., text, images) and **store** both the **original data** and **embeddings** in a **structured schema** thus providing efficient querying capabilities for similarity searches.
+
+Let's quickly [get started](./index.md) and learn how to manage embeddings in LanceDB. 
+
+## Bonus: As a developer, what you can create using embeddings?
+
+As a developer, you can create a variety of innovative applications using vector embeddings. Check out the following - 
+
+<div class="grid cards" markdown>
+
+-   __Chatbots__
+
+    ---
+
+    Develop chatbots that utilize embeddings to retrieve relevant context and generate coherent, contextually aware responses to user queries.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/chatbot.md)
+
+-   __Recommendation Systems__
+
+    ---
+
+    Develop systems that recommend content (such as articles, movies, or products) based on the similarity of keywords and descriptions, enhancing user experience.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/recommendersystem.md)
+
+-   __Vector Search__
+
+    ---
+
+    Build powerful applications that harness the full potential of semantic search, enabling them to retrieve relevant data quickly and effectively. 
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/vector_search.md)
+
+-   __RAG Applications__
+
+    ---
+
+    Combine the strengths of large language models (LLMs) with retrieval-based approaches to create more useful applications.
+
+    [:octicons-arrow-right-24: Check out examples](../examples/python_examples/rag.md)
+
+-   __Many more examples__
+
+    ---
+
+    Explore applied examples available as Colab notebooks or Python scripts to integrate into your applications.
+
+    [:octicons-arrow-right-24: More](../examples/examples_python.md)
+
+</div>
+
+
+
+
+
+
+
+

docs/src/examples/index.md

@@ -8,9 +8,15 @@ LanceDB provides language APIs, allowing you to embed a database in your languag
 * 👾 [JavaScript](examples_js.md) examples
 * 🦀 Rust examples (coming soon)
 
-## Applications powered by LanceDB
+## Python Applications powered by LanceDB
 
 | Project Name | Description |
 | --- | --- |
 | **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
 | **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |
+
+## Nodejs Applications powered by LanceDB
+
+| Project Name | Description |
+| --- | --- |
+| **Langchain Writing Assistant✍️ **<br>[![Github](../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/tree/main/applications/node/lanchain_writing_assistant) |  - **📂 Data Source Integration**:  Use your own data by specifying data source file, and the app instantly processes it to provide insights. <br>- **🧠 Intelligent Suggestions**:  Powered by LangChain.js and LanceDB, it improves writing productivity and accuracy.  <br>- **💡 Enhanced Writing Experience**: It delivers real-time contextual insights and factual suggestions while the user writes. |

docs/src/fts.md

@@ -2,7 +2,7 @@
 
 LanceDB provides support for full-text search via Lance (before via [Tantivy](https://github.com/quickwit-oss/tantivy) (Python only)), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
 
-Currently, the Lance full text search is missing some features that are in the Tantivy full text search. This includes phrase queries, re-ranking, and customizing the tokenizer. Thus, in Python, Tantivy is still the default way to do full text search and many of the instructions below apply just to Tantivy-based indices.
+Currently, the Lance full text search is missing some features that are in the Tantivy full text search. This includes query parser and customizing the tokenizer. Thus, in Python, Tantivy is still the default way to do full text search and many of the instructions below apply just to Tantivy-based indices.
 
 
 ## Installation (Only for Tantivy-based FTS)
@@ -205,7 +205,7 @@ table.create_fts_index(["text_field"], use_tantivy=True, ordering_field_names=["
 ## Phrase queries vs. terms queries
 
 !!! warning "Warn"
-    Lance-based FTS doesn't support queries combining by boolean operators `OR`, `AND`.
+    Lance-based FTS doesn't support queries using boolean operators `OR`, `AND`.
 
 For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
 or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms

docs/src/integrations/phidata.md

@@ -0,0 +1,383 @@
+**phidata** is a framework for building **AI Assistants** with long-term memory, contextual knowledge, and the ability to take actions using function calling. It helps turn general-purpose LLMs into specialized assistants tailored to your use case by extending its capabilities using **memory**, **knowledge**, and **tools**. 
+
+- **Memory**: Stores chat history in a **database** and enables LLMs to have long-term conversations.
+- **Knowledge**: Stores information in a **vector database** and provides LLMs with business context. (Here we will use LanceDB)
+- **Tools**: Enable LLMs to take actions like pulling data from an **API**, **sending emails** or **querying a database**, etc.
+
+![example](https://raw.githubusercontent.com/lancedb/assets/refs/heads/main/docs/assets/integration/phidata_assistant.png)
+
+Memory & knowledge make LLMs smarter while tools make them autonomous.
+
+LanceDB is a vector database and its integration into phidata makes it easy for us to provide a **knowledge base** to LLMs. It enables us to store information as [embeddings](../embeddings/understanding_embeddings.md) and search for the **results** similar to ours using **query**. 
+
+??? Question "What is Knowledge Base?"
+    Knowledge Base is a database of information that the Assistant can search to improve its responses. This information is stored in a vector database and provides LLMs with business context, which makes them respond in a context-aware manner.
+
+    While any type of storage can act as a knowledge base, vector databases offer the best solution for retrieving relevant results from dense information quickly.
+
+Let's see how using LanceDB inside phidata helps in making LLM more useful:
+
+## Prerequisites: install and import necessary dependencies
+
+**Create a virtual environment**
+
+1. install virtualenv package
+    ```python
+    pip install virtualenv
+    ```
+2. Create a directory for your project and go to the directory and create a virtual environment inside it.
+    ```python
+    mkdir phi
+    ```
+    ```python
+    cd phi
+    ```
+    ```python
+    python -m venv phidata_
+    ```
+
+**Activating virtual environment**
+
+1. from inside the project directory, run the following command to activate the virtual environment.
+    ```python
+    phidata_/Scripts/activate
+    ```
+
+**Install the following packages in the virtual environment**
+```python
+pip install lancedb phidata youtube_transcript_api openai ollama pandas numpy
+```
+
+**Create python files and import necessary libraries**
+
+You need to create two files - `transcript.py` and `ollama_assistant.py` or `openai_assistant.py`
+
+=== "openai_assistant.py"
+
+    ```python
+    import os, openai
+    from rich.prompt import Prompt
+    from phi.assistant import Assistant
+    from phi.knowledge.text import TextKnowledgeBase
+    from phi.vectordb.lancedb import LanceDb
+    from phi.llm.openai import OpenAIChat
+    from phi.embedder.openai import OpenAIEmbedder
+    from transcript import extract_transcript
+
+    if "OPENAI_API_KEY" not in os.environ:
+    # OR set the key here as a variable
+        openai.api_key = "sk-..."
+
+    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
+    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo" 
+    segment_duration = 20
+    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    from rich.prompt import Prompt
+    from phi.assistant import Assistant
+    from phi.knowledge.text import TextKnowledgeBase
+    from phi.vectordb.lancedb import LanceDb
+    from phi.llm.ollama import Ollama
+    from phi.embedder.ollama import OllamaEmbedder
+    from transcript import extract_transcript
+
+    # The code below creates a file "transcript.txt" in the directory, the txt file will be used below
+    youtube_url = "https://www.youtube.com/watch?v=Xs33-Gzl8Mo"
+    segment_duration = 20
+    transcript_text,dict_transcript = extract_transcript(youtube_url,segment_duration)
+    ```
+
+=== "transcript.py"
+
+    ``` python
+    from youtube_transcript_api import YouTubeTranscriptApi
+    import re
+
+    def smodify(seconds):
+        hours, remainder = divmod(seconds, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}"
+        
+    def extract_transcript(youtube_url,segment_duration):
+        # Extract video ID from the URL
+        video_id = re.search(r'(?<=v=)[\w-]+', youtube_url)
+        if not video_id:
+            video_id = re.search(r'(?<=be/)[\w-]+', youtube_url)
+        if not video_id:
+            return None
+
+        video_id = video_id.group(0)
+
+        # Attempt to fetch the transcript
+        try:
+            # Try to get the official transcript
+            transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=['en'])
+        except Exception:
+            # If no official transcript is found, try to get auto-generated transcript
+            try:
+                transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+                for transcript in transcript_list:
+                    transcript = transcript.translate('en').fetch()
+            except Exception:
+                return None
+
+        # Format the transcript into 120s chunks
+        transcript_text,dict_transcript = format_transcript(transcript,segment_duration)
+        # Open the file in write mode, which creates it if it doesn't exist
+        with open("transcript.txt", "w",encoding="utf-8") as file:
+            file.write(transcript_text)
+        return transcript_text,dict_transcript
+
+    def format_transcript(transcript,segment_duration):
+        chunked_transcript = []
+        chunk_dict = []
+        current_chunk = []
+        current_time = 0
+        # 2 minutes in seconds
+        start_time_chunk = 0  # To track the start time of the current chunk
+
+        for segment in transcript:
+            start_time = segment['start']
+            end_time_x = start_time + segment['duration']
+            text = segment['text']
+            
+            # Add text to the current chunk
+            current_chunk.append(text)
+            
+            # Update the current time with the duration of the current segment
+            # The duration of the current segment is given by segment['start'] - start_time_chunk
+            if current_chunk:
+                current_time = start_time - start_time_chunk
+            
+            # If current chunk duration reaches or exceeds 2 minutes, save the chunk
+            if current_time >= segment_duration:
+                # Use the start time of the first segment in the current chunk as the timestamp
+                chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
+                current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
+                chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
+                current_chunk = []  # Reset the chunk
+                start_time_chunk = start_time + segment['duration'] # Update the start time for the next chunk
+                current_time = 0  # Reset current time
+
+        # Add any remaining text in the last chunk
+        if current_chunk:
+            chunked_transcript.append(f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}] " + " ".join(current_chunk))
+            current_chunk = re.sub(r'[\xa0\n]', lambda x: '' if x.group() == '\xa0' else ' ', "\n".join(current_chunk))
+            chunk_dict.append({"timestamp":f"[{smodify(start_time_chunk)} to {smodify(end_time_x)}]", "text": "".join(current_chunk)})
+
+        return "\n\n".join(chunked_transcript), chunk_dict
+    ```
+
+!!! warning
+    If creating Ollama assistant, download and install Ollama [from here](https://ollama.com/) and then run the Ollama instance in the background. Also, download the required models using `ollama pull <model-name>`. Check out the models [here](https://ollama.com/library) 
+
+
+**Run the following command to deactivate the virtual environment if needed**
+```python
+deactivate
+```
+
+## **Step 1** - Create a Knowledge Base for AI Assistant using LanceDB
+
+=== "openai_assistant.py"
+
+    ```python
+    # Create knowledge Base with OpenAIEmbedder in LanceDB
+    knowledge_base = TextKnowledgeBase(
+        path="transcript.txt",
+        vector_db=LanceDb(
+            embedder=OpenAIEmbedder(api_key = openai.api_key),
+            table_name="transcript_documents",
+            uri="./t3mp/.lancedb",
+        ),
+        num_documents = 10
+    )
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    # Create knowledge Base with OllamaEmbedder in LanceDB
+    knowledge_base = TextKnowledgeBase(
+        path="transcript.txt",
+        vector_db=LanceDb(
+            embedder=OllamaEmbedder(model="nomic-embed-text",dimensions=768),
+            table_name="transcript_documents",
+            uri="./t2mp/.lancedb",
+        ),
+        num_documents = 10
+    )
+    ```
+Check out the list of **embedders** supported by **phidata** and their usage [here](https://docs.phidata.com/embedder/introduction).
+
+Here we have used `TextKnowledgeBase`, which loads text/docx files to the knowledge base.
+
+Let's see all the parameters that `TextKnowledgeBase` takes -
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`path`|`Union[str, Path]`| Path to text file(s). It can point to a single text file or a directory of text files.| provided by user |
+|`formats`|`List[str]`| File formats accepted by this knowledge base. |`[".txt"]`|
+|`vector_db`|`VectorDb`| Vector Database for the Knowledge Base. phidata provides a wrapper around many vector DBs, you can import it like this - `from phi.vectordb.lancedb import LanceDb` | provided by user |
+|`num_documents`|`int`| Number of results (documents/vectors) that vector search should return. |`5`|
+|`reader`|`TextReader`| phidata provides many types of reader objects which read data, clean it and create chunks of data, encapsulate each chunk inside an object of the `Document` class, and return **`List[Document]`**.  | `TextReader()` |
+|`optimize_on`|`int`| It is used to specify the number of documents on which to optimize the vector database. Supposed to create an index. |`1000`|
+
+??? Tip "Wonder! What is `Document` class?"
+    We know that, before storing the data in vectorDB, we need to split the data into smaller chunks upon which embeddings will be created and these embeddings along with the chunks will be stored in vectorDB. When the user queries over the vectorDB, some of these embeddings will be returned as the result based on the semantic similarity with the query.
+    
+    When the user queries over vectorDB, the queries are converted into embeddings, and a nearest neighbor search is performed over these query embeddings which returns the embeddings that correspond to most semantically similar chunks(parts of our data) present in vectorDB. 
+
+    Here, a “Document” is a class in phidata. Since there is an option to let phidata create and manage embeddings, it splits our data into smaller chunks(as expected). It does not directly create embeddings on it. Instead, it takes each chunk and encapsulates it inside the object of the `Document` class along with various other metadata related to the chunk. Then embeddings are created on these `Document` objects and stored in vectorDB.
+
+    ```python
+    class Document(BaseModel):
+        """Model for managing a document"""
+
+        content: str # <--- here data of chunk is stored 
+        id: Optional[str] = None
+        name: Optional[str] = None
+        meta_data: Dict[str, Any] = {}
+        embedder: Optional[Embedder] = None
+        embedding: Optional[List[float]] = None
+        usage: Optional[Dict[str, Any]] = None
+    ```
+
+However, using phidata you can load many other types of data in the knowledge base(other than text). Check out [phidata Knowledge Base](https://docs.phidata.com/knowledge/introduction) for more information.
+
+Let's dig deeper into the `vector_db` parameter and see what parameters `LanceDb` takes -
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`embedder`|`Embedder`| phidata provides many Embedders that abstract the interaction with embedding APIs and utilize it to generate embeddings. Check out other embedders [here](https://docs.phidata.com/embedder/introduction) | `OpenAIEmbedder` |
+|`distance`|`List[str]`| The choice of distance metric used to calculate the similarity between vectors, which directly impacts search results and performance in vector databases. |`Distance.cosine`|
+|`connection`|`lancedb.db.LanceTable`| LanceTable can be accessed through `.connection`. You can connect to an existing table of LanceDB, created outside of phidata, and utilize it. If not provided, it creates a new table using `table_name` parameter and adds it to `connection`. |`None`|
+|`uri`|`str`| It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. | `"/tmp/lancedb"` |
+|`table_name`|`str`|  If `connection` is not provided, it initializes and connects to a new **LanceDB table** with a specified(or default) name in the database present at `uri`. |`"phi"`|
+|`nprobes`|`int`| It refers to the number of partitions that the search algorithm examines to find the nearest neighbors of a given query vector. Higher values will yield better recall (more likely to find vectors if they exist) at the expense of latency. |`20`|
+
+
+!!! note
+    Since we just initialized the KnowledgeBase. The VectorDB table that corresponds to this Knowledge Base is not yet populated with our data. It will be populated in **Step 3**, once we perform the `load` operation. 
+    
+    You can check the state of the LanceDB table using - `knowledge_base.vector_db.connection.to_pandas()`
+
+Now that the Knowledge Base is initialized, , we can go to **step 2**.
+
+## **Step 2** -  Create an assistant with our choice of LLM and reference to the knowledge base.
+
+
+=== "openai_assistant.py"
+
+    ```python
+    # define an assistant with gpt-4o-mini llm and reference to the knowledge base created above
+    assistant = Assistant(
+        llm=OpenAIChat(model="gpt-4o-mini", max_tokens=1000, temperature=0.3,api_key = openai.api_key),
+        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
+        
+        This is transcript for the above timestamp: {relevant_document}
+        The user input is: {user_input}
+        generate highlights only when asked.
+        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
+        [timestamp] - highlight 1
+        [timestamp] - highlight 2
+        ... so on
+        
+        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
+        knowledge_base=knowledge_base,
+        add_references_to_prompt=True,
+    )
+    ```
+
+=== "ollama_assistant.py"
+
+    ```python
+    # define an assistant with llama3.1 llm and reference to the knowledge base created above
+    assistant = Assistant(
+        llm=Ollama(model="llama3.1"),
+        description="""You are an Expert in explaining youtube video transcripts. You are a bot that takes transcript of a video and answer the question based on it.
+
+        This is transcript for the above timestamp: {relevant_document}
+        The user input is: {user_input}
+        generate highlights only when asked.
+        When asked to generate highlights from the video, understand the context for each timestamp and create key highlight points, answer in following way - 
+        [timestamp] - highlight 1
+        [timestamp] - highlight 2
+        ... so on
+
+        Your task is to understand the user question, and provide an answer using the provided contexts. Your answers are correct, high-quality, and written by an domain expert. If the provided context does not contain the answer, simply state,'The provided context does not have the answer.'""",
+        knowledge_base=knowledge_base,
+        add_references_to_prompt=True,
+    )
+    ```
+
+Assistants add **memory**, **knowledge**, and **tools** to LLMs. Here we will add only **knowledge** in this example. 
+
+Whenever we will give a query to LLM, the assistant will retrieve relevant information from our **Knowledge Base**(table in LanceDB) and pass it to LLM along with the user query in a structured way. 
+
+- The `add_references_to_prompt=True` always adds information from the knowledge base to the prompt, regardless of whether it is relevant to the question.
+
+To know more about an creating assistant in phidata, check out [phidata docs](https://docs.phidata.com/assistants/introduction) here.
+
+## **Step 3** - Load data to Knowledge Base.
+
+```python
+# load out data into the knowledge_base (populating the LanceTable)
+assistant.knowledge_base.load(recreate=False)
+```
+The above code loads the data to the Knowledge Base(LanceDB Table) and now it is ready to be used by the assistant. 
+
+| Name| Type | Purpose | Default |
+|:----|:-----|:--------|:--------|
+|`recreate`|`bool`| If True, it drops the existing table and recreates the table in the vectorDB. |`False`|
+|`upsert`|`bool`| If True and the vectorDB supports upsert, it will upsert documents to the vector db. | `False` |
+|`skip_existing`|`bool`| If True, skips documents that already exist in the vectorDB when inserting. |`True`|
+
+??? tip "What is upsert?"
+    Upsert is a database operation that combines "update" and "insert". It updates existing records if a document with the same identifier does exist, or inserts new records if no matching record exists. This is useful for maintaining the most current information without manually checking for existence.
+
+During the Load operation, phidata directly interacts with the LanceDB library and performs the loading of the table with our data in the following steps -     
+
+1. **Creates** and **initializes** the table if it does not exist.
+
+2. Then it **splits** our data into smaller **chunks**.
+
+    ??? question "How do they create chunks?"
+        **phidata** provides many types of **Knowledge Bases** based on the type of data. Most of them :material-information-outline:{ title="except LlamaIndexKnowledgeBase and LangChainKnowledgeBase"} has a property method called `document_lists` of type `Iterator[List[Document]]`. During the load operation, this property method is invoked. It traverses on the data provided by us (in this case, a text file(s)) using `reader`. Then it **reads**, **creates chunks**, and **encapsulates** each chunk inside a `Document` object and yields **lists of `Document` objects** that contain our data.
+
+3. Then **embeddings** are created on these chunks are **inserted** into the LanceDB Table 
+
+    ??? question "How do they insert your data as different rows in LanceDB Table?"
+        The chunks of your data are in the form - **lists of `Document` objects**. It was yielded in the step above.
+
+        for each `Document` in `List[Document]`, it does the following operations:
+        
+        - Creates embedding on `Document`.
+        - Cleans the **content attribute**(chunks of our data is here) of `Document`.
+        - Prepares data by creating `id` and loading `payload` with the metadata related to this chunk. (1)
+            { .annotate }
+
+            1.  Three columns will be added to the table - `"id"`, `"vector"`, and `"payload"` (payload contains various metadata including **`content`**)
+                
+        - Then add this data to LanceTable. 
+
+4. Now the internal state of `knowledge_base` is changed (embeddings are created and loaded in the table ) and it **ready to be used by assistant**.
+
+## **Step 4** - Start a cli chatbot with access to the Knowledge base  
+
+```python
+# start cli chatbot with knowledge base
+assistant.print_response("Ask me about something from the knowledge base")
+while True:
+    message = Prompt.ask(f"[bold] :sunglasses: User [/bold]")
+    if message in ("exit", "bye"):
+        break
+    assistant.print_response(message, markdown=True)
+```  
+
+
+For more information and amazing cookbooks of phidata, read the [phidata documentation](https://docs.phidata.com/introduction) and also visit [LanceDB x phidata docmentation](https://docs.phidata.com/vectordb/lancedb).

docs/src/integrations/voxel51.md

@@ -1,13 +1,73 @@
 # FiftyOne
 
-FiftyOne is an open source toolkit for building high-quality datasets and computer vision models. It provides an API to create LanceDB tables and run similarity queries, both programmatically in Python and via point-and-click in the App.
+FiftyOne is an open source toolkit that enables users to curate better data and build better models. It includes tools for data exploration, visualization, and management, as well as features for collaboration and sharing. 
+
+Any developers, data scientists, and researchers who work with computer vision and machine learning can use FiftyOne to improve the quality of their datasets and deliver insights about their models.
+
 
 ![example](../assets/voxel.gif)
 
-## Basic recipe
+**FiftyOne** provides an API to create LanceDB tables and run similarity queries, both **programmatically in Python** and via **point-and-click in the App**.
 
-The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne
+Let's get started and see how to use **LanceDB** to create a **similarity index** on your FiftyOne datasets.
-datasets:
+
+## Overview 
+
+**[Embeddings](../embeddings/understanding_embeddings.md)** are foundational to all of the **vector search** features. In FiftyOne, embeddings are managed by the [**FiftyOne Brain**](https://docs.voxel51.com/user_guide/brain.html) that provides powerful machine learning techniques designed to transform how you curate your data from an art into a measurable science. 
+
+!!!question "Have you ever wanted to find the images most similar to an image in your dataset?"
+    The **FiftyOne Brain** makes computing **visual similarity** really easy. You can compute the similarity of samples in your dataset using an embedding model and store the results in the **brain key**. 
+
+    You can then sort your samples by similarity or use this information to find potential duplicate images.
+
+Here we will be doing the following : 
+
+1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. 
+
+    -  In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB). 
+    
+    !!!tip
+        You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes.
+
+2. **Query** - Once you have generated your similarity index, you can query your dataset with `sort_by_similarity()`. The query can be any of the following:
+
+    - An ID (sample or patch)
+    - A query vector of same dimension as the index
+    - A list of IDs (samples or patches)
+    - A text prompt (search semantically)
+
+## Prerequisites: install necessary dependencies
+
+1. **Create and activate a virtual environment**
+
+    Install virtualenv package and run the following command in your project directory.
+    ```python
+    python -m venv fiftyone_
+    ```
+    From inside the project directory run the following to activate the virtual environment.
+    === "Windows"
+
+        ```python
+        fiftyone_/Scripts/activate
+        ```
+
+    === "macOS/Linux"
+
+        ```python
+        source fiftyone_/Scripts/activate
+        ```
+
+2. **Install the following packages in the virtual environment**
+
+    To install FiftyOne, ensure you have activated any virtual environment that you are using, then run
+    ```python
+    pip install fiftyone
+    ```
+
+
+## Understand basic workflow
+
+The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne datasets:
 
 1. Load a dataset into FiftyOne.
 
@@ -19,14 +79,10 @@ datasets:
 
 5. If desired, delete the table.
 
-The example below demonstrates this workflow.
+## Quick Example
 
-!!! Note
+Let's jump on a quick example that demonstrates this workflow.
 
-    Install the LanceDB Python client to run the code shown below.
-    ```
-    pip install lancedb
-    ```
 
 ```python
 
@@ -36,7 +92,10 @@ import fiftyone.zoo as foz
 
 # Step 1: Load your data into FiftyOne
 dataset = foz.load_zoo_dataset("quickstart")
+```
+Make sure you install torch ([guide here](https://pytorch.org/get-started/locally/)) before proceeding. 
 
+```python
 # Steps 2 and 3: Compute embeddings and create a similarity index
 lancedb_index = fob.compute_similarity(
     dataset, 
@@ -45,8 +104,11 @@ lancedb_index = fob.compute_similarity(
     backend="lancedb",
 )
 ```
-Once the similarity index has been generated, we can query our data in FiftyOne
+
-by specifying the `brain_key`:
+!!! note
+    Running the code above will download the clip model (2.6Gb)
+
+Once the similarity index has been generated, we can query our data in FiftyOne by specifying the `brain_key`:
 
 ```python
 # Step 4: Query your data
@@ -56,7 +118,22 @@ view = dataset.sort_by_similarity(
     brain_key="lancedb_index",
     k=10,  # limit to 10 most similar samples
 )
+```
+The returned result are of type - `DatasetView`.
 
+!!! note
+    `DatasetView` does not hold its contents in-memory. Views simply store the rule(s) that are applied to extract the content of interest from the underlying Dataset when the view is iterated/aggregated on.
+
+    This means, for example, that the contents of a `DatasetView` may change as the underlying Dataset is modified.
+
+??? question "Can you query a view instead of dataset?"
+    Yes, you can also query a view.
+
+    Performing a similarity search on a `DatasetView` will only return results from the view; if the view contains samples that were not included in the index, they will never be included in the result.
+
+    This means that you can index an entire Dataset once and then perform searches on subsets of the dataset by constructing views that contain the images of interest.
+
+```python
 # Step 5 (optional): Cleanup
 
 # Delete the LanceDB table
@@ -66,4 +143,90 @@ lancedb_index.cleanup()
 dataset.delete_brain_run("lancedb_index")
 ```
 
+
+## Using LanceDB backend
+By default, calling `compute_similarity()` or `sort_by_similarity()` will use an sklearn backend.
+
+To use the LanceDB backend, simply set the optional `backend` parameter of `compute_similarity()` to `"lancedb"`:
+
+```python
+import fiftyone.brain as fob
+#... rest of the code
+fob.compute_similarity(..., backend="lancedb", ...)
+```
+
+Alternatively, you can configure FiftyOne to use the LanceDB backend by setting the following environment variable.
+
+In your terminal, set the environment variable using:
+=== "Windows"
+
+    ```python
+    $Env:FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND="lancedb" //powershell
+
+    set FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb //cmd
+    ```
+
+=== "macOS/Linux"
+
+    ```python
+    export FIFTYONE_BRAIN_DEFAULT_SIMILARITY_BACKEND=lancedb
+    ```
+
+!!! note
+    This will only run during the terminal session. Once terminal is closed, environment variable is deleted.
+
+Alternatively, you can **permanently** configure FiftyOne to use the LanceDB backend creating a `brain_config.json` at `~/.fiftyone/brain_config.json`. The JSON file may contain any desired subset of config fields that you wish to customize.
+
+```json
+{
+    "default_similarity_backend": "lancedb"
+}
+```
+This will override the default `brain_config` and will set it according to your customization. You can check the configuration by running the following code : 
+
+```python
+import fiftyone.brain as fob
+# Print your current brain config
+print(fob.brain_config)
+```
+
+## LanceDB config parameters
+
+The LanceDB backend supports query parameters that can be used to customize your similarity queries. These parameters include:
+
+| Name| Purpose | Default |
+|:----|:--------|:--------|
+|**table_name**|The name of the LanceDB table to use. If none is provided, a new table will be created|`None`|
+|**metric**|The embedding distance metric to use when creating a new table. The supported values are ("cosine", "euclidean")|`"cosine"`|
+|**uri**| The database URI to use. In this Database URI, tables will be created. |`"/tmp/lancedb"`|
+
+There are two ways to specify/customize the parameters:
+
+1. **Using `brain_config.json` file** 
+
+    ```json
+    {
+        "similarity_backends": {
+            "lancedb": {
+                "table_name": "your-table",
+                "metric": "euclidean",
+                "uri": "/tmp/lancedb"
+            }
+        }
+    }
+    ```
+
+2. **Directly passing to `compute_similarity()` to configure a specific new index** : 
+
+    ```python
+    lancedb_index = fob.compute_similarity(
+        ...
+        backend="lancedb",
+        brain_key="lancedb_index",
+        table_name="your-table",
+        metric="euclidean",
+        uri="/tmp/lancedb",
+    )
+    ```
+
 For a much more in depth walkthrough of the integration, visit the LanceDB x Voxel51 [docs page](https://docs.voxel51.com/integrations/lancedb.html).

docs/src/js/classes/Index.md

@@ -68,3 +68,25 @@ currently is also a memory intensive operation.
 #### Returns
 
 [`Index`](Index.md)
+
+### fts()
+
+> `static` **fts**(`options`?): [`Index`](Index.md)
+
+Create a full text search index
+
+This index is used to search for text data.  The index is created by tokenizing the text
+into words and then storing occurrences of these words in a data structure called inverted index
+that allows for fast search.
+
+During a search the query is tokenized and the inverted index is used to find the rows that
+contain the query words.  The rows are then scored based on BM25 and the top scoring rows are
+sorted and returned.
+
+#### Parameters
+
+• **options?**: `Partial`<[`FtsOptions`](../interfaces/FtsOptions.md)>
+
+#### Returns
+
+[`Index`](Index.md)

docs/src/js/classes/Table.md

@@ -501,16 +501,28 @@ Get the schema of the table.
 
 #### search(query)
 
-> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
+> `abstract` **search**(`query`, `queryType`, `ftsColumns`): [`VectorQuery`](VectorQuery.md)
 
 Create a search query to find the nearest neighbors
-of the given query vector
+of the given query vector, or the documents
+with the highest relevance to the query string.
 
 ##### Parameters
 
 • **query**: `string`
 
-the query. This will be converted to a vector using the table's provided embedding function
+the query. This will be converted to a vector using the table's provided embedding function,
+or the query string for full-text search if `queryType` is "fts".
+
+• **queryType**: `string` =  `"auto"` \| `"fts"`
+
+the type of query to run. If "auto", the query type will be determined based on the query.
+
+• **ftsColumns**: `string[] | str` =  undefined
+
+the columns to search in. If not provided, all indexed columns will be searched.
+
+For now, this can support to search only one column.
 
 ##### Returns
 

docs/src/js/globals.md

@@ -37,6 +37,7 @@
 - [IndexOptions](interfaces/IndexOptions.md)
 - [IndexStatistics](interfaces/IndexStatistics.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
+- [FtsOptions](interfaces/FtsOptions.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
 - [WriteOptions](interfaces/WriteOptions.md)

docs/src/rag/adaptive_rag.md

@@ -0,0 +1,51 @@
+**Adaptive RAG 🤹‍♂️**
+====================================================================
+Adaptive RAG introduces a RAG technique that combines query analysis with self-corrective RAG. 
+
+For Query Analysis, it uses a small classifier(LLM), to decide the query’s complexity. Query Analysis helps routing smoothly to adjust between different retrieval strategies No retrieval, Single-shot RAG or Iterative RAG.
+
+**[Official Paper](https://arxiv.org/pdf/2403.14403)**
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/adaptive_rag.png)
+  <figcaption>Adaptive-RAG: <a href="https://github.com/starsuzi/Adaptive-RAG">Source</a>
+  </figcaption>
+</figure>
+
+**[Offical Implementation](https://github.com/starsuzi/Adaptive-RAG)**
+
+Here’s a code snippet for query analysis
+
+```python
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.pydantic_v1 import BaseModel, Field
+from langchain_openai import ChatOpenAI
+
+class RouteQuery(BaseModel):
+    """Route a user query to the most relevant datasource."""
+
+    datasource: Literal["vectorstore", "web_search"] = Field(
+        ...,
+        description="Given a user question choose to route it to web search or a vectorstore.",
+    )
+
+
+# LLM with function call
+llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
+structured_llm_router = llm.with_structured_output(RouteQuery)
+```
+
+For defining and querying retriever
+
+```python
+# add documents in LanceDB
+vectorstore = LanceDB.from_documents(
+    documents=doc_splits,
+    embedding=OpenAIEmbeddings(),
+)
+retriever = vectorstore.as_retriever()
+
+# query using defined retriever
+question = "How adaptive RAG works"
+docs = retriever.get_relevant_documents(question)
+```

docs/src/rag/advanced_techniques/flare.md

@@ -0,0 +1,38 @@
+**FLARE 💥**
+====================================================================
+FLARE, stands for Forward-Looking Active REtrieval augmented generation is a generic retrieval-augmented generation method that actively decides when and what to retrieve using a prediction of the upcoming sentence to anticipate future content and utilize it as the query to retrieve relevant documents if it contains low-confidence tokens.
+
+**[Official Paper](https://arxiv.org/abs/2305.06983)**
+
+<figure markdown="span">
+  ![flare](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/flare.gif)
+  <figcaption>FLARE: <a href="https://github.com/jzbjyb/FLARE">Source</a></figcaption>
+</figure>
+
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)
+
+Here’s a code snippet for using FLARE with Langchain
+
+```python
+from langchain.vectorstores import LanceDB
+from langchain.document_loaders import ArxivLoader
+from langchain.chains import FlareChain
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+from langchain.llms import OpenAI
+
+llm = OpenAI()
+
+# load dataset
+
+# LanceDB retriever
+vector_store = LanceDB.from_documents(doc_chunks, embeddings, connection=table)
+retriever = vector_store.as_retriever()
+
+# define flare chain
+flare = FlareChain.from_llm(llm=llm,retriever=vector_store_retriever,max_generation_len=300,min_prob=0.45)
+
+result = flare.run(input_text)
+```
+
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)

docs/src/rag/advanced_techniques/hyde.md

@@ -0,0 +1,55 @@
+**HyDE: Hypothetical Document Embeddings 🤹‍♂️**
+====================================================================
+HyDE, stands for Hypothetical Document Embeddings is an approach used for precise zero-shot dense retrieval without relevance labels. It focuses on augmenting and improving similarity searches, often intertwined with vector stores in information retrieval. The method generates a hypothetical document for an incoming query, which is then embedded and used to look up real documents that are similar to the hypothetical document.
+
+**[Official Paper](https://arxiv.org/pdf/2212.10496)**
+
+<figure markdown="span">
+  ![hyde](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/hyde.png)
+  <figcaption>HyDE: <a href="https://arxiv.org/pdf/2212.10496">Source</a></figcaption>
+</figure>
+
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb)
+
+Here’s a code snippet for using HyDE with Langchain
+
+```python
+from langchain.llms import OpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain, HypotheticalDocumentEmbedder
+from langchain.vectorstores import LanceDB
+
+# set OPENAI_API_KEY as env variable before this step
+# initialize LLM and embedding function
+llm = OpenAI()
+emebeddings = OpenAIEmbeddings()
+
+# HyDE embedding
+embeddings = HypotheticalDocumentEmbedder(llm_chain=llm_chain,base_embeddings=embeddings)
+
+# load dataset
+
+# LanceDB retriever
+retriever = LanceDB.from_documents(documents, embeddings, connection=table)
+
+# prompt template
+prompt_template = """
+As a knowledgeable and helpful research assistant, your task is to provide informative answers based on the given context. Use your extensive knowledge base to offer clear, concise, and accurate responses to the user's inquiries.
+if quetion is not related to documents simply say you dont know
+Question: {question}
+
+Answer:
+"""
+
+prompt = PromptTemplate(input_variables=["question"], template=prompt_template)
+
+# LLM Chain
+llm_chain = LLMChain(llm=llm, prompt=prompt)
+
+# vector search
+retriever.similarity_search(query)
+llm_chain.run(query)
+```
+
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb)

docs/src/rag/agentic_rag.md

@@ -0,0 +1,101 @@
+**Agentic RAG 🤖**
+====================================================================
+Agentic RAG is Agent-based RAG introduces an advanced framework for answering questions by using intelligent agents instead of just relying on large language models. These agents act like expert researchers, handling complex tasks such as detailed planning, multi-step reasoning, and using external tools. They navigate multiple documents, compare information, and generate accurate answers. This system is easily scalable, with each new document set managed by a sub-agent, making it a powerful tool for tackling a wide range of information needs.
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/agentic_rag.png)
+  <figcaption>Agent-based RAG</figcaption>
+</figure>
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)
+
+Here’s a code snippet for defining retriever using Langchain
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.vectorstores import LanceDB
+from langchain_openai import OpenAIEmbeddings
+
+urls = [
+    "https://content.dgft.gov.in/Website/CIEP.pdf",
+    "https://content.dgft.gov.in/Website/GAE.pdf",
+    "https://content.dgft.gov.in/Website/HTE.pdf",
+]
+
+
+docs = [WebBaseLoader(url).load() for url in urls]
+docs_list = [item for sublist in docs for item in sublist]
+
+text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+    chunk_size=100, chunk_overlap=50
+)
+doc_splits = text_splitter.split_documents(docs_list)
+
+# add documents in LanceDB
+vectorstore = LanceDB.from_documents(
+    documents=doc_splits,
+    embedding=OpenAIEmbeddings(),
+)
+retriever = vectorstore.as_retriever()
+
+```
+
+Agent that formulates an improved query for better retrieval results and then grades the retrieved documents
+
+```python
+def grade_documents(state) -> Literal["generate", "rewrite"]:
+    class grade(BaseModel):
+        binary_score: str = Field(description="Relevance score 'yes' or 'no'")
+
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+    llm_with_tool = model.with_structured_output(grade)
+    prompt = PromptTemplate(
+        template="""You are a grader assessing relevance of a retrieved document to a user question. \n
+        Here is the retrieved document: \n\n {context} \n\n
+        Here is the user question: {question} \n
+        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \n
+        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.""",
+        input_variables=["context", "question"],
+    )
+    chain = prompt | llm_with_tool
+
+    messages = state["messages"]
+    last_message = messages[-1]
+    question = messages[0].content
+    docs = last_message.content
+
+    scored_result = chain.invoke({"question": question, "context": docs})
+    score = scored_result.binary_score
+
+    return "generate" if score == "yes" else "rewrite"
+
+
+def agent(state):
+    messages = state["messages"]
+    model = ChatOpenAI(temperature=0, streaming=True, model="gpt-4-turbo")
+    model = model.bind_tools(tools)
+    response = model.invoke(messages)
+    return {"messages": [response]}
+
+
+def rewrite(state):
+    messages = state["messages"]
+    question = messages[0].content
+    msg = [
+        HumanMessage(
+            content=f""" \n
+            Look at the input and try to reason about the underlying semantic intent / meaning. \n
+            Here is the initial question:
+            \n ------- \n
+            {question}
+            \n ------- \n
+            Formulate an improved question: """,
+        )
+    ]
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+    response = model.invoke(msg)
+    return {"messages": [response]}
+```
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)

docs/src/rag/corrective_rag.md

@@ -0,0 +1,120 @@
+**Corrective RAG ✅**
+====================================================================
+
+Corrective-RAG (CRAG) is a strategy for Retrieval-Augmented Generation (RAG) that includes self-reflection and self-grading of retrieved documents. Here’s a simplified breakdown of the steps involved:
+
+1. **Relevance Check**: If at least one document meets the relevance threshold, the process moves forward to the generation phase.
+2. **Knowledge Refinement**: Before generating an answer, the process refines the knowledge by dividing the document into smaller segments called "knowledge strips."
+3. **Grading and Filtering**: Each "knowledge strip" is graded, and irrelevant ones are filtered out.
+4. **Additional Data Source**: If all documents are below the relevance threshold, or if the system is unsure about their relevance, it will seek additional information by performing a web search to supplement the retrieved data.
+
+Above steps are mentioned in 
+**[Official Paper](https://arxiv.org/abs/2401.15884)**
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/crag_paper.png)
+  <figcaption>Corrective RAG: <a href="https://github.com/HuskyInSalt/CRAG">Source</a>
+  </figcaption>
+</figure>
+
+Corrective Retrieval-Augmented Generation (CRAG) is a method that works like a **built-in fact-checker**.
+
+**[Offical Implementation](https://github.com/HuskyInSalt/CRAG)**
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)
+
+Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), and retrieves the relevant documents.
+
+```python
+import pandas as pd
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+class Docs(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+table = db.create_table("docs", schema=Docs)
+
+# considering chunks are in list format
+df = pd.DataFrame({'text':chunks})
+table.add(data=df)
+
+# as per document feeded
+query = "How Transformers work?" 
+actual = table.search(query).limit(1).to_list()[0]
+print(actual.text)
+```
+
+Code snippet for grading retrieved documents, filtering out irrelevant ones, and performing a web search if necessary:
+
+```python
+def grade_documents(state):
+    """
+        Determines whether the retrieved documents are relevant to the question
+
+        Args:
+            state (dict): The current graph state
+
+        Returns:
+            state (dict): Updates documents key with relevant documents
+        """
+
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = state_dict["documents"]
+
+    class grade(BaseModel):
+        """
+            Binary score for relevance check
+        """
+
+        binary_score: str = Field(description="Relevance score 'yes' or 'no'")
+
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+    # grading using openai
+    grade_tool_oai = convert_to_openai_tool(grade)
+    llm_with_tool = model.bind(
+        tools=[convert_to_openai_tool(grade_tool_oai)],
+        tool_choice={"type": "function", "function": {"name": "grade"}},
+    )
+
+    parser_tool = PydanticToolsParser(tools=[grade])
+    prompt = PromptTemplate(
+        template="""You are a grader assessing relevance of a retrieved document to a user question. \n
+        Here is the retrieved document: \n\n {context} \n\n
+        Here is the user question: {question} \n
+        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \n
+        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.""",
+        input_variables=["context", "question"],
+    )
+
+    chain = prompt | llm_with_tool | parser_tool
+
+    filtered_docs = []
+    search = "No" 
+    for d in documents:
+        score = chain.invoke({"question": question, "context": d.page_content})
+        grade = score[0].binary_score
+        if grade == "yes":
+            filtered_docs.append(d)
+        else:
+            search = "Yes" 
+            continue
+
+    return {
+        "keys": {
+            "documents": filtered_docs,
+            "question": question,
+            "run_web_search": search,
+        }
+    }
+```
+
+Check Colab for the Implementation of CRAG with Langgraph 
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)

docs/src/rag/graph_rag.md

@@ -0,0 +1,54 @@
+**Graph RAG 📊**
+====================================================================
+Graph RAG uses knowledge graphs together with large language models (LLMs) to improve how information is retrieved and generated. It overcomes the limits of traditional search methods by using knowledge graphs, which organize data as connected entities and relationships.
+
+One of the main benefits of Graph RAG is its ability to capture and represent complex relationships between entities, something that traditional text-based retrieval systems struggle with. By using this structured knowledge, LLMs can better grasp the context and details of a query, resulting in more accurate and insightful answers.
+
+**[Official Paper](https://arxiv.org/pdf/2404.16130)**
+
+**[Offical Implementation](https://github.com/microsoft/graphrag)**
+
+[Microsoft Research Blog](https://www.microsoft.com/en-us/research/blog/graphrag-unlocking-llm-discovery-on-narrative-private-data/)
+
+!!! note "Default VectorDB"
+
+    Graph RAG uses LanceDB as the default vector database for performing vector search to retrieve relevant entities.
+
+Working with Graph RAG is quite straightforward
+
+- **Installation and API KEY as env variable**
+
+Set `OPENAI_API_KEY` as `GRAPHRAG_API_KEY`
+
+```bash
+pip install graphrag
+export GRAPHRAG_API_KEY="sk-..."
+```
+
+- **Initial structure for indexing dataset**
+
+```bash
+python3 -m graphrag.index --init --root dataset-dir
+```
+
+- **Index Dataset**
+
+```bash
+python3 -m graphrag.index --root dataset-dir
+```
+
+- **Execute Query**
+
+Global Query Execution gives a broad overview of dataset
+
+```bash
+python3 -m graphrag.query --root dataset-dir --method global "query-question"
+```
+
+Local Query Execution gives a detailed and specific answers based on the context of the entities
+
+```bash
+python3 -m graphrag.query --root  dataset-dir --method local "query-question"
+```
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Graphrag/main.ipynb)

docs/src/rag/multi_head_rag.md

@@ -0,0 +1,49 @@
+**Multi-Head RAG 📃**
+====================================================================
+
+Multi-head RAG (MRAG) is designed to handle queries that need multiple documents with diverse content. These queries are tough because the documents’ embeddings can be far apart, making retrieval difficult. MRAG simplifies this by using the activations from a Transformer's multi-head attention layer, rather than the decoder layer, to fetch these varied documents. Different attention heads capture different aspects of the data, so using these activations helps create embeddings that better represent various data facets and improves retrieval accuracy for complex queries.
+
+**[Official Paper](https://arxiv.org/pdf/2406.05085)**
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/mrag-paper.png)
+  <figcaption>Multi-Head RAG: <a href="https://github.com/spcl/MRAG">Source</a>
+  </figcaption>
+</figure>
+
+MRAG is cost-effective and energy-efficient because it avoids extra LLM queries, multiple model instances, increased storage, and additional inference passes.
+
+**[Official Implementation](https://github.com/spcl/MRAG)**
+
+Here’s a code snippet for defining different embedding spaces with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/)
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+# model definition using LanceDB Embedding API
+model1 = get_registry().get("openai").create()
+model2 = get_registry().get("ollama").create(name="llama3")
+model3 = get_registry().get("ollama").create(name="mistral")
+
+
+# define schema for creating embedding spaces with Embedding API
+class Space1(LanceModel):
+    text: str = model1.SourceField()
+    vector: Vector(model1.ndims()) = model1.VectorField()
+
+
+class Space2(LanceModel):
+    text: str = model2.SourceField()
+    vector: Vector(model2.ndims()) = model2.VectorField()
+
+
+class Space3(LanceModel):
+    text: str = model3.SourceField()
+    vector: Vector(model3.ndims()) = model3.VectorField()
+```
+
+Create different tables using defined embedding spaces, then make queries to each embedding space. Use the resulted closest documents from each embedding space to generate answers.
+
+

docs/src/rag/self_rag.md

@@ -0,0 +1,96 @@
+**Self RAG 🤳**
+====================================================================
+Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better retrieved information, generated text, and  checking their own work, all without losing their flexibility. Unlike the traditional Retrieval-Augmented Generation (RAG) method, Self-RAG retrieves information as needed, can skip retrieval if not needed, and evaluates its own output while generating text. It also uses a process to pick the best output based on different preferences.
+
+**[Official Paper](https://arxiv.org/pdf/2310.11511)**
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/self_rag.png)
+  <figcaption>Self RAG: <a href="https://github.com/AkariAsai/self-rag">Source</a>
+  </figcaption>
+</figure>
+
+**[Offical Implementation](https://github.com/AkariAsai/self-rag)**
+
+Self-RAG starts by generating a response without retrieving extra info if it's not needed. For questions that need more details, it retrieves to get the necessary information.
+
+Here’s a code snippet for defining retriever using Langchain
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.vectorstores import LanceDB
+from langchain_openai import OpenAIEmbeddings
+
+urls = [
+    "https://lilianweng.github.io/posts/2023-06-23-agent/",
+    "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/",
+    "https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/",
+]
+
+
+docs = [WebBaseLoader(url).load() for url in urls]
+docs_list = [item for sublist in docs for item in sublist]
+
+text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+    chunk_size=100, chunk_overlap=50
+)
+doc_splits = text_splitter.split_documents(docs_list)
+
+# add documents in LanceDB
+vectorstore = LanceDB.from_documents(
+    documents=doc_splits,
+    embedding=OpenAIEmbeddings(),
+)
+retriever = vectorstore.as_retriever()
+
+```
+
+Functions that grades the retrieved documents and if required formulates an improved query for better retrieval results  
+
+```python
+def grade_documents(state) -> Literal["generate", "rewrite"]:
+    class grade(BaseModel):
+        binary_score: str = Field(description="Relevance score 'yes' or 'no'")
+
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+    llm_with_tool = model.with_structured_output(grade)
+    prompt = PromptTemplate(
+        template="""You are a grader assessing relevance of a retrieved document to a user question. \n
+        Here is the retrieved document: \n\n {context} \n\n
+        Here is the user question: {question} \n
+        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \n
+        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.""",
+        input_variables=["context", "question"],
+    )
+    chain = prompt | llm_with_tool
+
+    messages = state["messages"]
+    last_message = messages[-1]
+    question = messages[0].content
+    docs = last_message.content
+
+    scored_result = chain.invoke({"question": question, "context": docs})
+    score = scored_result.binary_score
+
+    return "generate" if score == "yes" else "rewrite"
+
+
+def rewrite(state):
+    messages = state["messages"]
+    question = messages[0].content
+    msg = [
+        HumanMessage(
+            content=f""" \n
+            Look at the input and try to reason about the underlying semantic intent / meaning. \n
+            Here is the initial question:
+            \n ------- \n
+            {question}
+            \n ------- \n
+            Formulate an improved question: """,
+        )
+    ]
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+    response = model.invoke(msg)
+    return {"messages": [response]}
+```

docs/src/rag/sfr_rag.md

@@ -0,0 +1,17 @@
+**SFR RAG 📑**
+====================================================================
+Salesforce AI Research introduces SFR-RAG, a 9-billion-parameter language model trained with a significant emphasis on reliable, precise, and faithful contextual generation abilities specific to real-world RAG use cases and relevant agentic tasks. They include precise factual knowledge extraction, distinguishing relevant against distracting contexts, citing appropriate sources along with answers, producing complex and multi-hop reasoning over multiple contexts, consistent format following, as well as refraining from hallucination over unanswerable queries.
+
+**[Offical Implementation](https://github.com/SalesforceAIResearch/SFR-RAG)**
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/salesforce_contextbench.png)
+  <figcaption>Average Scores in ContextualBench: <a href="https://blog.salesforceairesearch.com/sfr-rag/">Source</a>
+  </figcaption>
+</figure>
+
+To reliably evaluate LLMs in contextual question-answering for RAG, Saleforce introduced [ContextualBench](https://huggingface.co/datasets/Salesforce/ContextualBench?ref=blog.salesforceairesearch.com), featuring 7 benchmarks like [HotpotQA](https://arxiv.org/abs/1809.09600?ref=blog.salesforceairesearch.com) and [2WikiHopQA](https://www.aclweb.org/anthology/2020.coling-main.580/?ref=blog.salesforceairesearch.com) with consistent setups. 
+
+SFR-RAG outperforms GPT-4o, achieving state-of-the-art results in 3 out of 7 benchmarks, and significantly surpasses Command-R+ while using 10 times fewer parameters. It also excels at handling context, even when facts are altered or conflicting.
+
+[Saleforce AI Research Blog](https://blog.salesforceairesearch.com/sfr-rag/)

docs/src/rag/vanilla_rag.md

@@ -0,0 +1,54 @@
+**Vanilla RAG 🌱**
+====================================================================
+
+RAG(Retrieval-Augmented Generation) works by finding documents related to the user's question, combining them with a prompt for a large language model (LLM), and then using the LLM to create more accurate and relevant answers.
+
+Here’s a simple guide to building a RAG pipeline from scratch:
+
+1. **Data Loading**: Gather and load the documents you want to use for answering questions.
+
+2. **Chunking and Embedding**: Split the documents into smaller chunks and convert them into numerical vectors (embeddings) that capture their meaning.
+
+3. **Vector Store**: Create a LanceDB table to store and manage these vectors for quick access during retrieval.
+
+4. **Retrieval & Prompt Preparation**: When a question is asked, find the most relevant document chunks from the table and prepare a prompt combining these chunks with the question.
+
+5. **Answer Generation**: Send the prepared prompt to a LLM to generate a detailed and accurate answer.
+
+<figure markdown="span">
+  ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/rag_from_scratch.png)
+  <figcaption>Vanilla RAG
+  </figcaption>
+</figure>
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-from-Scratch/RAG_from_Scratch.ipynb)
+
+Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), which simplifies the process by handling embedding extraction and querying in one step.
+
+```python
+import pandas as pd
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+class Docs(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+table = db.create_table("docs", schema=Docs)
+
+# considering chunks are in list format
+df = pd.DataFrame({'text':chunks})
+table.add(data=df)
+
+query = "What is issue date of lease?"
+actual = table.search(query).limit(1).to_list()[0]
+print(actual.text)
+```
+
+Check Colab for the complete code
+
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-from-Scratch/RAG_from_Scratch.ipynb)

docs/src/reranking/linear_combination.md

@@ -1,6 +1,9 @@
 # Linear Combination Reranker
 
-This is the default re-ranker used by LanceDB hybrid search. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+!!! note
+    This is depricated. It is recommended to use the `RRFReranker` instead, if you want to use a score based reranker.
+
+It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
 
 !!! note
     Supported Query Types: Hybrid

docs/src/reranking/rrf.md

@@ -1,6 +1,6 @@
 # Reciprocal Rank Fusion Reranker
 
-Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
+This is the default re-ranker used by LanceDB hybrid search. Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
 
 
 !!! note

docs/test/md_testing.py

@@ -20,7 +20,11 @@ excluded_globs = [
     "../src/reranking/*.md",
     "../src/guides/tuning_retrievers/*.md",
     "../src/embeddings/available_embedding_models/text_embedding_functions/*.md",
-    "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md"
+    "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md",
+    "../src/rag/*.md",
+    "../src/rag/advanced_techniques/*.md"
+
+
 ]
 
 python_prefix = "py"

java/core/lancedb-jni/Cargo.toml

@@ -2,7 +2,7 @@
 name = "lancedb-jni"
 description = "JNI bindings for LanceDB"
 # TODO modify lancedb/Cargo.toml for version and dependencies
-version = "0.4.18"
+version = "0.10.0"
 edition.workspace = true
 repository.workspace = true
 readme.workspace = true

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.0.3</version>
+        <version>0.11.0-beta.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -44,7 +44,7 @@
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>
-             <scope>test</scope>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.0.3</version>
+    <version>0.11.0-beta.1</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>
@@ -92,7 +92,7 @@
         </repository>
     </distributionManagement>
 
-     <build>
+    <build>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -167,7 +167,8 @@
                     <version>3.2.5</version>
                     <configuration>
                         <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
-                        <forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
+                        <forkNode
+                            implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory" />
                         <useSystemClassLoader>false</useSystemClassLoader>
                     </configuration>
                 </plugin>
@@ -183,7 +184,7 @@
         </pluginManagement>
     </build>
 
-   <profiles>
+    <profiles>
         <profile>
             <id>jdk8</id>
             <activation>
@@ -210,7 +211,8 @@
                         <version>3.2.5</version>
                         <configuration>
                             <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
-                            <forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory" />
+                            <forkNode
+                                implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory" />
                             <useSystemClassLoader>false</useSystemClassLoader>
                         </configuration>
                     </plugin>

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.10.0-beta.1",
+  "version": "0.11.0-beta.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -58,7 +58,7 @@
     "ts-node-dev": "^2.0.0",
     "typedoc": "^0.24.7",
     "typedoc-plugin-markdown": "^3.15.3",
-    "typescript": "*",
+    "typescript": "^5.1.0",
     "uuid": "^9.0.0"
   },
   "dependencies": {
@@ -88,10 +88,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.20",
+    "@lancedb/vectordb-darwin-arm64": "0.11.0-beta.1",
-    "@lancedb/vectordb-darwin-x64": "0.4.20",
+    "@lancedb/vectordb-darwin-x64": "0.11.0-beta.1",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.20",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.11.0-beta.1",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.20",
+    "@lancedb/vectordb-linux-x64-gnu": "0.11.0-beta.1",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.20"
+    "@lancedb/vectordb-win32-x64-msvc": "0.11.0-beta.1"
   }
 }

node/src/index.ts

@@ -60,7 +60,7 @@ export {
   type MakeArrowTableOptions
 } from "./arrow";
 
-const defaultAwsRegion = "us-west-2";
+const defaultAwsRegion = "us-east-1";
 
 const defaultRequestTimeout = 10_000
 
@@ -111,7 +111,7 @@ export interface ConnectionOptions {
    */
   apiKey?: string
 
-  /** Region to connect */
+  /** Region to connect. Default is 'us-east-1' */
   region?: string
 
   /**
@@ -197,28 +197,32 @@ export async function connect(
 export async function connect(
   arg: string | Partial<ConnectionOptions>
 ): Promise<Connection> {
-  let opts: ConnectionOptions;
+  let partOpts: Partial<ConnectionOptions>;
   if (typeof arg === "string") {
-    opts = { uri: arg };
+    partOpts = { uri: arg };
   } else {
     const keys = Object.keys(arg);
     if (keys.length === 1 && keys[0] === "uri" && typeof arg.uri === "string") {
-      opts = { uri: arg.uri };
+      partOpts = { uri: arg.uri };
     } else {
-      opts = Object.assign(
+      partOpts = arg;
-        {
-          uri: "",
-          awsCredentials: undefined,
-          awsRegion: defaultAwsRegion,
-          apiKey: undefined,
-          region: defaultAwsRegion,
-          timeout: defaultRequestTimeout 
-        },
-        arg
-      );
     }
   }
 
+  let defaultRegion = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION;
+  defaultRegion = (defaultRegion ?? "").trim() !== "" ? defaultRegion : defaultAwsRegion;
+
+  const opts: ConnectionOptions = {
+    uri: partOpts.uri ?? "",
+    awsCredentials: partOpts.awsCredentials ?? undefined,
+    awsRegion: partOpts.awsRegion ?? defaultRegion,
+    apiKey: partOpts.apiKey ?? undefined,
+    region: partOpts.region ?? defaultRegion,
+    timeout: partOpts.timeout ?? defaultRequestTimeout,
+    readConsistencyInterval: partOpts.readConsistencyInterval ?? undefined,
+    storageOptions: partOpts.storageOptions ?? undefined,
+    hostOverride: partOpts.hostOverride ?? undefined
+  }
   if (opts.uri.startsWith("db://")) {
     // Remote connection
     return new RemoteConnection(opts);
@@ -720,9 +724,9 @@ export interface VectorIndex {
 export interface IndexStats {
   numIndexedRows: number | null
   numUnindexedRows: number | null
-  indexType: string | null
+  indexType: string
-  distanceType: string | null
+  distanceType?: string
-  completedAt: string | null
+  numIndices?: number
 }
 
 /**

node/src/integration_test/test.ts

@@ -14,6 +14,7 @@
 
 import { describe } from 'mocha'
 import * as chai from 'chai'
+import { assert } from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 import { v4 as uuidv4 } from 'uuid'
 
@@ -22,7 +23,6 @@ import { tmpdir } from 'os'
 import * as fs from 'fs'
 import * as path from 'path'
 
-const assert = chai.assert
 chai.use(chaiAsPromised)
 
 describe('LanceDB AWS Integration test', function () {

node/src/query.ts

@@ -33,6 +33,7 @@ export class Query<T = number[]> {
   private _filter?: string
   private _metricType?: MetricType
   private _prefilter: boolean
+  private _fastSearch: boolean
   protected readonly _embeddings?: EmbeddingFunction<T>
 
   constructor (query?: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
@@ -46,6 +47,7 @@ export class Query<T = number[]> {
     this._metricType = undefined
     this._embeddings = embeddings
     this._prefilter = false
+    this._fastSearch = false
   }
 
   /***
@@ -110,6 +112,15 @@ export class Query<T = number[]> {
     return this
   }
 
+  /**
+   * Skip searching un-indexed data. This can make search faster, but will miss
+   * any data that is not yet indexed.
+   */
+  fastSearch (value: boolean): Query<T> {
+    this._fastSearch = value
+    return this
+  }
+
   /**
      * Execute the query and return the results as an Array of Objects
      */
@@ -131,9 +142,9 @@ export class Query<T = number[]> {
       Object.keys(entry).forEach((key: string) => {
         if (entry[key] instanceof Vector) {
           // toJSON() returns f16 array correctly
-          newObject[key] = (entry[key] as Vector).toJSON()
+          newObject[key] = (entry[key] as any).toJSON()
         } else {
-          newObject[key] = entry[key]
+          newObject[key] = entry[key] as any
         }
       })
       return newObject as unknown as T

node/src/remote/client.ts

@@ -17,6 +17,7 @@ import axios, { type AxiosResponse, type ResponseType } from 'axios'
 import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
 
 import { type RemoteResponse, type RemoteRequest, Method } from '../middleware'
+import type { MetricType } from '..'
 
 interface HttpLancedbClientMiddleware {
   onRemoteRequest(
@@ -82,7 +83,7 @@ async function callWithMiddlewares (
 
 interface MiddlewareInvocationOptions {
   responseType?: ResponseType
-  timeout?: number,
+  timeout?: number
 }
 
 /**
@@ -130,8 +131,8 @@ export class HttpLancedbClient {
     url: string,
     apiKey: string,
     timeout?: number,
-    private readonly _dbName?: string,
+    private readonly _dbName?: string
-    
+
   ) {
     this._url = url
     this._apiKey = () => apiKey
@@ -151,7 +152,9 @@ export class HttpLancedbClient {
     prefilter: boolean,
     refineFactor?: number,
     columns?: string[],
-    filter?: string
+    filter?: string,
+    metricType?: MetricType,
+    fastSearch?: boolean
   ): Promise<ArrowTable<any>> {
     const result = await this.post(
       `/v1/table/${tableName}/query/`,
@@ -159,10 +162,12 @@ export class HttpLancedbClient {
         vector,
         k,
         nprobes,
-        refineFactor,
+        refine_factor: refineFactor,
         columns,
         filter,
-        prefilter
+        prefilter,
+        metric: metricType,
+        fast_search: fastSearch
       },
       undefined,
       undefined,
@@ -237,7 +242,7 @@ export class HttpLancedbClient {
     try {
       response = await callWithMiddlewares(req, this._middlewares, {
         responseType,
-        timeout: this._timeout,
+        timeout: this._timeout
       })
 
       // return response

node/src/remote/index.ts

@@ -238,16 +238,18 @@ export class RemoteQuery<T = number[]> extends Query<T> {
       (this as any)._prefilter,
       (this as any)._refineFactor,
       (this as any)._select,
-      (this as any)._filter
+      (this as any)._filter,
+      (this as any)._metricType,
+      (this as any)._fastSearch
     )
 
     return data.toArray().map((entry: Record<string, unknown>) => {
       const newObject: Record<string, unknown> = {}
       Object.keys(entry).forEach((key: string) => {
         if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
+          newObject[key] = (entry[key] as any).toArray()
         } else {
-          newObject[key] = entry[key]
+          newObject[key] = entry[key] as any
         }
       })
       return newObject as unknown as T
@@ -524,8 +526,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       numIndexedRows: body?.num_indexed_rows,
       numUnindexedRows: body?.num_unindexed_rows,
       indexType: body?.index_type,
-      distanceType: body?.distance_type,
+      distanceType: body?.distance_type
-      completedAt: body?.completed_at
     }
   }
 

node/src/test/test.ts

@@ -14,6 +14,7 @@
 
 import { describe } from "mocha";
 import { track } from "temp";
+import { assert, expect } from 'chai'
 import * as chai from "chai";
 import * as chaiAsPromised from "chai-as-promised";
 
@@ -44,8 +45,6 @@ import {
 } from "apache-arrow";
 import type { RemoteRequest, RemoteResponse } from "../middleware";
 
-const expect = chai.expect;
-const assert = chai.assert;
 chai.use(chaiAsPromised);
 
 describe("LanceDB client", function () {
@@ -112,8 +111,8 @@ describe("LanceDB client", function () {
           name: 'name_2',
           price: 10,
           is_active: true,
-          vector: [ 0, 0.1 ]
+          vector: [0, 0.1]
-        },
+        }
       ]);
       assert.equal(await table2.countRows(), 3);
     });
@@ -169,7 +168,7 @@ describe("LanceDB client", function () {
 
       // Should reject a bad filter
       await expect(table.filter("id % 2 = 0 AND").execute()).to.be.rejectedWith(
-        /.*sql parser error: Expected an expression:, found: EOF.*/
+        /.*sql parser error: .*/
       );
     });
 
@@ -888,9 +887,12 @@ describe("LanceDB client", function () {
       expect(indices[0].columns).to.have.lengthOf(1);
       expect(indices[0].columns[0]).to.equal("vector");
 
-      const stats = await table.indexStats(indices[0].uuid);
+      const stats = await table.indexStats(indices[0].name);
       expect(stats.numIndexedRows).to.equal(300);
       expect(stats.numUnindexedRows).to.equal(0);
+      expect(stats.indexType).to.equal("IVF_PQ");
+      expect(stats.distanceType).to.equal("l2");
+      expect(stats.numIndices).to.equal(1);
     }).timeout(50_000);
   });
 

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.0.0"
+version = "0.11.0-beta.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -14,7 +14,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-ipc.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb" }
+lancedb = { path = "../rust/lancedb", features = ["remote"] }
 napi = { version = "2.16.8", default-features = false, features = [
     "napi9",
     "async",

nodejs/__test__/connection.test.ts

@@ -107,7 +107,7 @@ describe("given a connection", () => {
     const data = [...Array(10000).keys()].map((i) => ({ id: i }));
 
     // Create in v1 mode
-    let table = await db.createTable("test", data);
+    let table = await db.createTable("test", data, { useLegacyFormat: true });
 
     const isV2 = async (table: Table) => {
       const data = await table.query().toArrow({ maxBatchLength: 100000 });
@@ -118,7 +118,7 @@ describe("given a connection", () => {
     await expect(isV2(table)).resolves.toBe(false);
 
     // Create in v2 mode
-    table = await db.createTable("test_v2", data, { useLegacyFormat: false });
+    table = await db.createTable("test_v2", data);
 
     await expect(isV2(table)).resolves.toBe(true);
 

nodejs/__test__/remote.test.ts

@@ -0,0 +1,93 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as http from "http";
+import { RequestListener } from "http";
+import { Connection, ConnectionOptions, connect } from "../lancedb";
+
+async function withMockDatabase(
+  listener: RequestListener,
+  callback: (db: Connection) => void,
+  connectionOptions?: ConnectionOptions,
+) {
+  const server = http.createServer(listener);
+  server.listen(8000);
+
+  const db = await connect(
+    "db://dev",
+    Object.assign(
+      {
+        apiKey: "fake",
+        hostOverride: "http://localhost:8000",
+      },
+      connectionOptions,
+    ),
+  );
+
+  try {
+    await callback(db);
+  } finally {
+    server.close();
+  }
+}
+
+describe("remote connection", () => {
+  it("should accept partial connection options", async () => {
+    await connect("db://test", {
+      apiKey: "fake",
+      clientConfig: {
+        timeoutConfig: { readTimeout: 5 },
+        retryConfig: { retries: 2 },
+      },
+    });
+  });
+
+  it("should pass down apiKey and userAgent", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.headers["x-api-key"]).toEqual("fake");
+        expect(req.headers["user-agent"]).toEqual(
+          `LanceDB-Node-Client/${process.env.npm_package_version}`,
+        );
+
+        const body = JSON.stringify({ tables: [] });
+        res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+      },
+      async (db) => {
+        const tableNames = await db.tableNames();
+        expect(tableNames).toEqual([]);
+      },
+    );
+  });
+
+  it("allows customizing user agent", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.headers["user-agent"]).toEqual("MyApp/1.0");
+
+        const body = JSON.stringify({ tables: [] });
+        res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+      },
+      async (db) => {
+        const tableNames = await db.tableNames();
+        expect(tableNames).toEqual([]);
+      },
+      {
+        clientConfig: {
+          userAgent: "MyApp/1.0",
+        },
+      },
+    );
+  });
+});

nodejs/__test__/table.test.ts

@@ -479,6 +479,9 @@ describe("When creating an index", () => {
     expect(stats).toBeDefined();
     expect(stats?.numIndexedRows).toEqual(300);
     expect(stats?.numUnindexedRows).toEqual(0);
+    expect(stats?.distanceType).toBeUndefined();
+    expect(stats?.indexType).toEqual("BTREE");
+    expect(stats?.numIndices).toEqual(1);
   });
 
   test("when getting stats on non-existent index", async () => {
@@ -872,7 +875,7 @@ describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
       ];
       const table = await db.createTable("test", data);
       await table.createIndex("text", {
-        config: Index.fts({ withPositions: false }),
+        config: Index.fts({ withPosition: false }),
       });
 
       const results = await table.search("hello").toArray();

nodejs/lancedb/connection.ts

@@ -44,11 +44,12 @@ export interface CreateTableOptions {
    * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
    */
   storageOptions?: Record<string, string>;
+
   /**
    * The version of the data storage format to use.
    *
-   * The default is `legacy`, which is Lance format v1.
+   * The default is `stable`.
-   * `stable` is the new format, which is Lance format v2.
+   * Set to "legacy" to use the old format.
    */
   dataStorageVersion?: string;
 
@@ -64,9 +65,9 @@ export interface CreateTableOptions {
   /**
    * If true then data files will be written with the legacy format
    *
-   * The default is true while the new format is in beta
+   * The default is false.
    *
-   * Deprecated.
+   * Deprecated. Use data storage version instead.
    */
   useLegacyFormat?: boolean;
   schema?: SchemaLike;
@@ -266,7 +267,7 @@ export class LocalConnection extends Connection {
       throw new Error("data is required");
     }
     const { buf, mode } = await Table.parseTableData(data, options);
-    let dataStorageVersion = "legacy";
+    let dataStorageVersion = "stable";
     if (options?.dataStorageVersion !== undefined) {
       dataStorageVersion = options.dataStorageVersion;
     } else if (options?.useLegacyFormat !== undefined) {
@@ -303,7 +304,7 @@ export class LocalConnection extends Connection {
       metadata = registry.getTableMetadata([embeddingFunction]);
     }
 
-    let dataStorageVersion = "legacy";
+    let dataStorageVersion = "stable";
     if (options?.dataStorageVersion !== undefined) {
       dataStorageVersion = options.dataStorageVersion;
     } else if (options?.useLegacyFormat !== undefined) {

nodejs/lancedb/index.ts

@@ -23,8 +23,6 @@ import {
   Connection as LanceDbConnection,
 } from "./native.js";
 
-import { RemoteConnection, RemoteConnectionOptions } from "./remote";
-
 export {
   WriteOptions,
   WriteMode,
@@ -32,8 +30,10 @@ export {
   ColumnAlteration,
   ConnectionOptions,
   IndexStatistics,
-  IndexMetadata,
   IndexConfig,
+  ClientConfig,
+  TimeoutConfig,
+  RetryConfig,
 } from "./native.js";
 
 export {
@@ -88,7 +88,7 @@ export * as embedding from "./embedding";
  */
 export async function connect(
   uri: string,
-  opts?: Partial<ConnectionOptions | RemoteConnectionOptions>,
+  opts?: Partial<ConnectionOptions>,
 ): Promise<Connection>;
 /**
  * Connect to a LanceDB instance at the given URI.
@@ -109,13 +109,11 @@ export async function connect(
  * ```
  */
 export async function connect(
-  opts: Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string },
+  opts: Partial<ConnectionOptions> & { uri: string },
 ): Promise<Connection>;
 export async function connect(
-  uriOrOptions:
+  uriOrOptions: string | (Partial<ConnectionOptions> & { uri: string }),
-    | string
+  opts: Partial<ConnectionOptions> = {},
-    | (Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string }),
-  opts: Partial<ConnectionOptions | RemoteConnectionOptions> = {},
 ): Promise<Connection> {
   let uri: string | undefined;
   if (typeof uriOrOptions !== "string") {
@@ -130,9 +128,6 @@ export async function connect(
     throw new Error("uri is required");
   }
 
-  if (uri?.startsWith("db://")) {
-    return new RemoteConnection(uri, opts as RemoteConnectionOptions);
-  }
   opts = (opts as ConnectionOptions) ?? {};
   (<ConnectionOptions>opts).storageOptions = cleanseStorageOptions(
     (<ConnectionOptions>opts).storageOptions,

nodejs/lancedb/indices.ts

@@ -113,22 +113,218 @@ export interface IvfPqOptions {
   sampleRate?: number;
 }
 
+/**
+ * Options to create an `HNSW_PQ` index
+ */
 export interface HnswPqOptions {
+  /**
+   * The distance metric used to train the index.
+   *
+   * Default value is "l2".
+   *
+   * The following distance types are available:
+   *
+   * "l2" - Euclidean distance. This is a very common distance metric that
+   * accounts for both magnitude and direction when determining the distance
+   * between vectors. L2 distance has a range of [0, ∞).
+   *
+   * "cosine" - Cosine distance.  Cosine distance is a distance metric
+   * calculated from the cosine similarity between two vectors. Cosine
+   * similarity is a measure of similarity between two non-zero vectors of an
+   * inner product space. It is defined to equal the cosine of the angle
+   * between them.  Unlike L2, the cosine distance is not affected by the
+   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+   *
+   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   */
   distanceType?: "l2" | "cosine" | "dot";
+
+  /**
+   * The number of IVF partitions to create.
+   *
+   * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+   * well for most tables. For very large tables, training just one HNSW graph
+   * will require too much memory. Each partition becomes its own HNSW graph, so
+   * setting this value higher reduces the peak memory use of training.
+   *
+   */
   numPartitions?: number;
+
+  /**
+   * Number of sub-vectors of PQ.
+   *
+   * This value controls how much the vector is compressed during the quantization step.
+   * The more sub vectors there are the less the vector is compressed.  The default is
+   * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+   * by 16 we use the dimension divded by 8.
+   *
+   * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+   * us to use efficient SIMD instructions.
+   *
+   * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+   * will likely result in poor performance.
+   *
+   */
   numSubVectors?: number;
+
+  /**
+   * Max iterations to train kmeans.
+   *
+   * The default value is 50.
+   *
+   * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+   * controls how many iterations of kmeans to run.
+   *
+   * Increasing this might improve the quality of the index but in most cases the parameter
+   * is unused because kmeans will converge with fewer iterations.  The parameter is only
+   * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+   * that setting this larger will lead to the index converging anyways.
+   *
+   */
   maxIterations?: number;
+
+  /**
+   * The rate used to calculate the number of training vectors for kmeans.
+   *
+   * Default value is 256.
+   *
+   * When an IVF index is trained, we need to calculate partitions.  These are groups
+   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+   *
+   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+   * random sample of the data.  This parameter controls the size of the sample.  The total
+   * number of vectors used to train the index is `sample_rate * num_partitions`.
+   *
+   * Increasing this value might improve the quality of the index but in most cases the
+   * default should be sufficient.
+   *
+   */
   sampleRate?: number;
+
+  /**
+   * The number of neighbors to select for each vector in the HNSW graph.
+   *
+   * The default value is 20.
+   *
+   * This value controls the tradeoff between search speed and accuracy.
+   * The higher the value the more accurate the search but the slower it will be.
+   *
+   */
   m?: number;
+
+  /**
+   * The number of candidates to evaluate during the construction of the HNSW graph.
+   *
+   * The default value is 300.
+   *
+   * This value controls the tradeoff between build speed and accuracy.
+   * The higher the value the more accurate the build but the slower it will be.
+   * 150 to 300 is the typical range. 100 is a minimum for good quality search
+   * results. In most cases, there is no benefit to setting this higher than 500.
+   * This value should be set to a value that is not less than `ef` in the search phase.
+   *
+   */
   efConstruction?: number;
 }
 
+/**
+ * Options to create an `HNSW_SQ` index
+ */
 export interface HnswSqOptions {
+  /**
+   * The distance metric used to train the index.
+   *
+   * Default value is "l2".
+   *
+   * The following distance types are available:
+   *
+   * "l2" - Euclidean distance. This is a very common distance metric that
+   * accounts for both magnitude and direction when determining the distance
+   * between vectors. L2 distance has a range of [0, ∞).
+   *
+   * "cosine" - Cosine distance.  Cosine distance is a distance metric
+   * calculated from the cosine similarity between two vectors. Cosine
+   * similarity is a measure of similarity between two non-zero vectors of an
+   * inner product space. It is defined to equal the cosine of the angle
+   * between them.  Unlike L2, the cosine distance is not affected by the
+   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+   *
+   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+   */
   distanceType?: "l2" | "cosine" | "dot";
+
+  /**
+   * The number of IVF partitions to create.
+   *
+   * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+   * well for most tables. For very large tables, training just one HNSW graph
+   * will require too much memory. Each partition becomes its own HNSW graph, so
+   * setting this value higher reduces the peak memory use of training.
+   *
+   */
   numPartitions?: number;
+
+  /**
+   * Max iterations to train kmeans.
+   *
+   * The default value is 50.
+   *
+   * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+   * controls how many iterations of kmeans to run.
+   *
+   * Increasing this might improve the quality of the index but in most cases the parameter
+   * is unused because kmeans will converge with fewer iterations.  The parameter is only
+   * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+   * that setting this larger will lead to the index converging anyways.
+   *
+   */
   maxIterations?: number;
+
+  /**
+   * The rate used to calculate the number of training vectors for kmeans.
+   *
+   * Default value is 256.
+   *
+   * When an IVF index is trained, we need to calculate partitions.  These are groups
+   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+   *
+   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+   * random sample of the data.  This parameter controls the size of the sample.  The total
+   * number of vectors used to train the index is `sample_rate * num_partitions`.
+   *
+   * Increasing this value might improve the quality of the index but in most cases the
+   * default should be sufficient.
+   *
+   */
   sampleRate?: number;
+
+  /**
+   * The number of neighbors to select for each vector in the HNSW graph.
+   *
+   * The default value is 20.
+   *
+   * This value controls the tradeoff between search speed and accuracy.
+   * The higher the value the more accurate the search but the slower it will be.
+   *
+   */
   m?: number;
+
+  /**
+   * The number of candidates to evaluate during the construction of the HNSW graph.
+   *
+   * The default value is 300.
+   *
+   * This value controls the tradeoff between build speed and accuracy.
+   * The higher the value the more accurate the build but the slower it will be.
+   * 150 to 300 is the typical range. 100 is a minimum for good quality search
+   * results. In most cases, there is no benefit to setting this higher than 500.
+   * This value should be set to a value that is not less than `ef` in the search phase.
+   *
+   */
   efConstruction?: number;
 }
 
@@ -142,7 +338,7 @@ export interface FtsOptions {
    * If set to false, the index will not store the positions of the tokens in the text,
    * which will make the index smaller and faster to build, but will not support phrase queries.
    */
-  withPositions?: boolean;
+  withPosition?: boolean;
 }
 
 export class Index {
@@ -244,12 +440,16 @@ export class Index {
    * For now, the full text search index only supports English, and doesn't support phrase search.
    */
   static fts(options?: Partial<FtsOptions>) {
-    return new Index(LanceDbIndex.fts(options?.withPositions));
+    return new Index(LanceDbIndex.fts(options?.withPosition));
   }
 
   /**
    *
-   * Create a hnswpq index
+   * Create a hnswPq index
+   *
+   * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+   * It is a variant of the HNSW algorithm that uses product quantization to compress
+   * the vectors.
    *
    */
   static hnswPq(options?: Partial<HnswPqOptions>) {
@@ -268,7 +468,11 @@ export class Index {
 
   /**
    *
-   * Create a hnswsq index
+   * Create a hnswSq index
+   *
+   * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+   * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+   * the vectors.
    *
    */
   static hnswSq(options?: Partial<HnswSqOptions>) {

nodejs/lancedb/remote/client.ts

@@ -1,218 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import axios, {
-  AxiosError,
-  type AxiosResponse,
-  type ResponseType,
-} from "axios";
-import { Table as ArrowTable } from "../arrow";
-import { tableFromIPC } from "../arrow";
-import { VectorQuery } from "../query";
-
-export class RestfulLanceDBClient {
-  #dbName: string;
-  #region: string;
-  #apiKey: string;
-  #hostOverride?: string;
-  #closed: boolean = false;
-  #timeout: number = 12 * 1000; // 12 seconds;
-  #session?: import("axios").AxiosInstance;
-
-  constructor(
-    dbName: string,
-    apiKey: string,
-    region: string,
-    hostOverride?: string,
-    timeout?: number,
-  ) {
-    this.#dbName = dbName;
-    this.#apiKey = apiKey;
-    this.#region = region;
-    this.#hostOverride = hostOverride ?? this.#hostOverride;
-    this.#timeout = timeout ?? this.#timeout;
-  }
-
-  // todo: cache the session.
-  get session(): import("axios").AxiosInstance {
-    if (this.#session !== undefined) {
-      return this.#session;
-    } else {
-      return axios.create({
-        baseURL: this.url,
-        headers: {
-          // biome-ignore lint: external API
-          Authorization: `Bearer ${this.#apiKey}`,
-        },
-        transformResponse: decodeErrorData,
-        timeout: this.#timeout,
-      });
-    }
-  }
-
-  get url(): string {
-    return (
-      this.#hostOverride ??
-      `https://${this.#dbName}.${this.#region}.api.lancedb.com`
-    );
-  }
-
-  get headers(): { [key: string]: string } {
-    const headers: { [key: string]: string } = {
-      "x-api-key": this.#apiKey,
-      "x-request-id": "na",
-    };
-    if (this.#region == "local") {
-      headers["Host"] = `${this.#dbName}.${this.#region}.api.lancedb.com`;
-    }
-    if (this.#hostOverride) {
-      headers["x-lancedb-database"] = this.#dbName;
-    }
-    return headers;
-  }
-
-  isOpen(): boolean {
-    return !this.#closed;
-  }
-
-  private checkNotClosed(): void {
-    if (this.#closed) {
-      throw new Error("Connection is closed");
-    }
-  }
-
-  close(): void {
-    this.#session = undefined;
-    this.#closed = true;
-  }
-
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  async get(uri: string, params?: Record<string, any>): Promise<any> {
-    this.checkNotClosed();
-    uri = new URL(uri, this.url).toString();
-    let response;
-    try {
-      response = await this.session.get(uri, {
-        headers: this.headers,
-        params,
-      });
-    } catch (e) {
-      if (e instanceof AxiosError && e.response) {
-        response = e.response;
-      } else {
-        throw e;
-      }
-    }
-
-    RestfulLanceDBClient.checkStatus(response!);
-    return response!.data;
-  }
-
-  // biome-ignore lint/suspicious/noExplicitAny: api response
-  async post(uri: string, body?: any): Promise<any>;
-  async post(
-    uri: string,
-    // biome-ignore lint/suspicious/noExplicitAny: api request
-    body: any,
-    additional: {
-      config?: { responseType: "arraybuffer" };
-      headers?: Record<string, string>;
-      params?: Record<string, string>;
-    },
-  ): Promise<Buffer>;
-  async post(
-    uri: string,
-    // biome-ignore lint/suspicious/noExplicitAny: api request
-    body?: any,
-    additional?: {
-      config?: { responseType: ResponseType };
-      headers?: Record<string, string>;
-      params?: Record<string, string>;
-    },
-    // biome-ignore lint/suspicious/noExplicitAny: api response
-  ): Promise<any> {
-    this.checkNotClosed();
-    uri = new URL(uri, this.url).toString();
-    additional = Object.assign(
-      { config: { responseType: "json" } },
-      additional,
-    );
-
-    const headers = { ...this.headers, ...additional.headers };
-
-    if (!headers["Content-Type"]) {
-      headers["Content-Type"] = "application/json";
-    }
-    let response;
-    try {
-      response = await this.session.post(uri, body, {
-        headers,
-        responseType: additional!.config!.responseType,
-        params: new Map(Object.entries(additional.params ?? {})),
-      });
-    } catch (e) {
-      if (e instanceof AxiosError && e.response) {
-        response = e.response;
-      } else {
-        throw e;
-      }
-    }
-    RestfulLanceDBClient.checkStatus(response!);
-    if (additional!.config!.responseType === "arraybuffer") {
-      return response!.data;
-    } else {
-      return JSON.parse(response!.data);
-    }
-  }
-
-  async listTables(limit = 10, pageToken = ""): Promise<string[]> {
-    const json = await this.get("/v1/table", { limit, pageToken });
-    return json.tables;
-  }
-
-  async query(tableName: string, query: VectorQuery): Promise<ArrowTable> {
-    const tbl = await this.post(`/v1/table/${tableName}/query`, query, {
-      config: {
-        responseType: "arraybuffer",
-      },
-    });
-    return tableFromIPC(tbl);
-  }
-
-  static checkStatus(response: AxiosResponse): void {
-    if (response.status === 404) {
-      throw new Error(`Not found: ${response.data}`);
-    } else if (response.status >= 400 && response.status < 500) {
-      throw new Error(
-        `Bad Request: ${response.status}, error: ${response.data}`,
-      );
-    } else if (response.status >= 500 && response.status < 600) {
-      throw new Error(
-        `Internal Server Error: ${response.status}, error: ${response.data}`,
-      );
-    } else if (response.status !== 200) {
-      throw new Error(
-        `Unknown Error: ${response.status}, error: ${response.data}`,
-      );
-    }
-  }
-}
-
-function decodeErrorData(data: unknown) {
-  if (Buffer.isBuffer(data)) {
-    const decoded = data.toString("utf-8");
-    return decoded;
-  }
-  return data;
-}

nodejs/lancedb/remote/connection.ts

@@ -1,193 +0,0 @@
-import { Schema } from "apache-arrow";
-import {
-  Data,
-  SchemaLike,
-  fromTableToStreamBuffer,
-  makeEmptyTable,
-} from "../arrow";
-import {
-  Connection,
-  CreateTableOptions,
-  OpenTableOptions,
-  TableNamesOptions,
-} from "../connection";
-import { Table } from "../table";
-import { TTLCache } from "../util";
-import { RestfulLanceDBClient } from "./client";
-import { RemoteTable } from "./table";
-
-export interface RemoteConnectionOptions {
-  apiKey?: string;
-  region?: string;
-  hostOverride?: string;
-  timeout?: number;
-}
-
-export class RemoteConnection extends Connection {
-  #dbName: string;
-  #apiKey: string;
-  #region: string;
-  #client: RestfulLanceDBClient;
-  #tableCache = new TTLCache(300_000);
-
-  constructor(
-    url: string,
-    { apiKey, region, hostOverride, timeout }: RemoteConnectionOptions,
-  ) {
-    super();
-    apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
-    region = region ?? process.env.LANCEDB_REGION;
-
-    if (!apiKey) {
-      throw new Error("apiKey is required when connecting to LanceDB Cloud");
-    }
-
-    if (!region) {
-      throw new Error("region is required when connecting to LanceDB Cloud");
-    }
-
-    const parsed = new URL(url);
-    if (parsed.protocol !== "db:") {
-      throw new Error(
-        `invalid protocol: ${parsed.protocol}, only accepts db://`,
-      );
-    }
-
-    this.#dbName = parsed.hostname;
-    this.#apiKey = apiKey;
-    this.#region = region;
-    this.#client = new RestfulLanceDBClient(
-      this.#dbName,
-      this.#apiKey,
-      this.#region,
-      hostOverride,
-      timeout,
-    );
-  }
-
-  isOpen(): boolean {
-    return this.#client.isOpen();
-  }
-  close(): void {
-    return this.#client.close();
-  }
-
-  display(): string {
-    return `RemoteConnection(${this.#dbName})`;
-  }
-
-  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
-    const response = await this.#client.get("/v1/table/", {
-      limit: options?.limit ?? 10,
-      // biome-ignore lint/style/useNamingConvention: <explanation>
-      page_token: options?.startAfter ?? "",
-    });
-    const body = await response.body();
-    for (const table of body.tables) {
-      this.#tableCache.set(table, true);
-    }
-    return body.tables;
-  }
-
-  async openTable(
-    name: string,
-    _options?: Partial<OpenTableOptions> | undefined,
-  ): Promise<Table> {
-    if (this.#tableCache.get(name) === undefined) {
-      await this.#client.post(
-        `/v1/table/${encodeURIComponent(name)}/describe/`,
-      );
-      this.#tableCache.set(name, true);
-    }
-    return new RemoteTable(this.#client, name, this.#dbName);
-  }
-
-  async createTable(
-    nameOrOptions:
-      | string
-      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
-    data?: Data,
-    options?: Partial<CreateTableOptions> | undefined,
-  ): Promise<Table> {
-    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-      const { name, data, ...options } = nameOrOptions;
-      return this.createTable(name, data, options);
-    }
-    if (data === undefined) {
-      throw new Error("data is required");
-    }
-    if (options?.mode) {
-      console.warn(
-        "option 'mode' is not supported in LanceDB Cloud",
-        "LanceDB Cloud only supports the default 'create' mode.",
-        "If the table already exists, an error will be thrown.",
-      );
-    }
-    if (options?.embeddingFunction) {
-      console.warn(
-        "embedding_functions is not yet supported on LanceDB Cloud.",
-        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
-        "for this feature.",
-      );
-    }
-
-    const { buf } = await Table.parseTableData(
-      data,
-      options,
-      true /** streaming */,
-    );
-
-    await this.#client.post(
-      `/v1/table/${encodeURIComponent(nameOrOptions)}/create/`,
-      buf,
-      {
-        config: {
-          responseType: "arraybuffer",
-        },
-        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
-      },
-    );
-    this.#tableCache.set(nameOrOptions, true);
-    return new RemoteTable(this.#client, nameOrOptions, this.#dbName);
-  }
-
-  async createEmptyTable(
-    name: string,
-    schema: SchemaLike,
-    options?: Partial<CreateTableOptions> | undefined,
-  ): Promise<Table> {
-    if (options?.mode) {
-      console.warn(`mode is not supported on LanceDB Cloud`);
-    }
-
-    if (options?.embeddingFunction) {
-      console.warn(
-        "embeddingFunction is not yet supported on LanceDB Cloud.",
-        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
-        "for this feature.",
-      );
-    }
-    const emptyTable = makeEmptyTable(schema);
-    const buf = await fromTableToStreamBuffer(emptyTable);
-
-    await this.#client.post(
-      `/v1/table/${encodeURIComponent(name)}/create/`,
-      buf,
-      {
-        config: {
-          responseType: "arraybuffer",
-        },
-        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
-      },
-    );
-
-    this.#tableCache.set(name, true);
-    return new RemoteTable(this.#client, name, this.#dbName);
-  }
-
-  async dropTable(name: string): Promise<void> {
-    await this.#client.post(`/v1/table/${encodeURIComponent(name)}/drop/`);
-
-    this.#tableCache.delete(name);
-  }
-}

nodejs/lancedb/remote/index.ts

@@ -1,3 +0,0 @@
-export { RestfulLanceDBClient } from "./client";
-export { type RemoteConnectionOptions, RemoteConnection } from "./connection";
-export { RemoteTable } from "./table";

nodejs/lancedb/remote/table.ts

@@ -1,226 +0,0 @@
-// Copyright 2023 LanceDB Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Table as ArrowTable } from "apache-arrow";
-
-import { Data, IntoVector } from "../arrow";
-
-import { IndexStatistics } from "..";
-import { CreateTableOptions } from "../connection";
-import { IndexOptions } from "../indices";
-import { MergeInsertBuilder } from "../merge";
-import { VectorQuery } from "../query";
-import { AddDataOptions, Table, UpdateOptions } from "../table";
-import { IntoSql, toSQL } from "../util";
-import { RestfulLanceDBClient } from "./client";
-
-export class RemoteTable extends Table {
-  #client: RestfulLanceDBClient;
-  #name: string;
-
-  // Used in the display() method
-  #dbName: string;
-
-  get #tablePrefix() {
-    return `/v1/table/${encodeURIComponent(this.#name)}/`;
-  }
-
-  get name(): string {
-    return this.#name;
-  }
-
-  public constructor(
-    client: RestfulLanceDBClient,
-    tableName: string,
-    dbName: string,
-  ) {
-    super();
-    this.#client = client;
-    this.#name = tableName;
-    this.#dbName = dbName;
-  }
-
-  isOpen(): boolean {
-    return !this.#client.isOpen();
-  }
-
-  close(): void {
-    this.#client.close();
-  }
-
-  display(): string {
-    return `RemoteTable(${this.#dbName}; ${this.#name})`;
-  }
-
-  async schema(): Promise<import("apache-arrow").Schema> {
-    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
-    // TODO: parse this into a valid arrow schema
-    return resp.schema;
-  }
-  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
-    const { buf, mode } = await Table.parseTableData(
-      data,
-      options as CreateTableOptions,
-      true,
-    );
-    await this.#client.post(`${this.#tablePrefix}/insert/`, buf, {
-      params: {
-        mode,
-      },
-      headers: {
-        "Content-Type": "application/vnd.apache.arrow.stream",
-      },
-    });
-  }
-
-  async update(
-    optsOrUpdates:
-      | (Map<string, string> | Record<string, string>)
-      | ({
-          values: Map<string, IntoSql> | Record<string, IntoSql>;
-        } & Partial<UpdateOptions>)
-      | ({
-          valuesSql: Map<string, string> | Record<string, string>;
-        } & Partial<UpdateOptions>),
-    options?: Partial<UpdateOptions>,
-  ): Promise<void> {
-    const isValues =
-      "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
-    const isValuesSql =
-      "valuesSql" in optsOrUpdates &&
-      typeof optsOrUpdates.valuesSql !== "string";
-    const isMap = (obj: unknown): obj is Map<string, string> => {
-      return obj instanceof Map;
-    };
-
-    let predicate;
-    let columns: [string, string][];
-    switch (true) {
-      case isMap(optsOrUpdates):
-        columns = Array.from(optsOrUpdates.entries());
-        predicate = options?.where;
-        break;
-      case isValues && isMap(optsOrUpdates.values):
-        columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
-          k,
-          toSQL(v),
-        ]);
-        predicate = optsOrUpdates.where;
-        break;
-      case isValues && !isMap(optsOrUpdates.values):
-        columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
-          k,
-          toSQL(v),
-        ]);
-        predicate = optsOrUpdates.where;
-        break;
-
-      case isValuesSql && isMap(optsOrUpdates.valuesSql):
-        columns = Array.from(optsOrUpdates.valuesSql.entries());
-        predicate = optsOrUpdates.where;
-        break;
-      case isValuesSql && !isMap(optsOrUpdates.valuesSql):
-        columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
-          k,
-          v,
-        ]);
-        predicate = optsOrUpdates.where;
-        break;
-      default:
-        columns = Object.entries(optsOrUpdates as Record<string, string>);
-        predicate = options?.where;
-    }
-
-    await this.#client.post(`${this.#tablePrefix}/update/`, {
-      predicate: predicate ?? null,
-      updates: columns,
-    });
-  }
-  async countRows(filter?: unknown): Promise<number> {
-    const payload = { predicate: filter };
-    return await this.#client.post(`${this.#tablePrefix}/count_rows/`, payload);
-  }
-
-  async delete(predicate: unknown): Promise<void> {
-    const payload = { predicate };
-    await this.#client.post(`${this.#tablePrefix}/delete/`, payload);
-  }
-  async createIndex(
-    column: string,
-    options?: Partial<IndexOptions>,
-  ): Promise<void> {
-    if (options !== undefined) {
-      console.warn("options are not yet supported on the LanceDB cloud");
-    }
-    const indexType = "vector";
-    const metric = "L2";
-    const data = {
-      column,
-      // biome-ignore lint/style/useNamingConvention: external API
-      index_type: indexType,
-      // biome-ignore lint/style/useNamingConvention: external API
-      metric_type: metric,
-    };
-    await this.#client.post(`${this.#tablePrefix}/create_index`, data);
-  }
-  query(): import("..").Query {
-    throw new Error("query() is not yet supported on the LanceDB cloud");
-  }
-
-  search(_query: string | IntoVector): VectorQuery {
-    throw new Error("search() is not yet supported on the LanceDB cloud");
-  }
-  vectorSearch(_vector: unknown): import("..").VectorQuery {
-    throw new Error("vectorSearch() is not yet supported on the LanceDB cloud");
-  }
-  addColumns(_newColumnTransforms: unknown): Promise<void> {
-    throw new Error("addColumns() is not yet supported on the LanceDB cloud");
-  }
-  alterColumns(_columnAlterations: unknown): Promise<void> {
-    throw new Error("alterColumns() is not yet supported on the LanceDB cloud");
-  }
-  dropColumns(_columnNames: unknown): Promise<void> {
-    throw new Error("dropColumns() is not yet supported on the LanceDB cloud");
-  }
-  async version(): Promise<number> {
-    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
-    return resp.version;
-  }
-  checkout(_version: unknown): Promise<void> {
-    throw new Error("checkout() is not yet supported on the LanceDB cloud");
-  }
-  checkoutLatest(): Promise<void> {
-    throw new Error(
-      "checkoutLatest() is not yet supported on the LanceDB cloud",
-    );
-  }
-  restore(): Promise<void> {
-    throw new Error("restore() is not yet supported on the LanceDB cloud");
-  }
-  optimize(_options?: unknown): Promise<import("../native").OptimizeStats> {
-    throw new Error("optimize() is not yet supported on the LanceDB cloud");
-  }
-  async listIndices(): Promise<import("../native").IndexConfig[]> {
-    return await this.#client.post(`${this.#tablePrefix}/index/list/`);
-  }
-  toArrow(): Promise<ArrowTable> {
-    throw new Error("toArrow() is not yet supported on the LanceDB cloud");
-  }
-  mergeInsert(_on: string | string[]): MergeInsertBuilder {
-    throw new Error("mergeInsert() is not yet supported on the LanceDB cloud");
-  }
-  async indexStats(_name: string): Promise<IndexStatistics | undefined> {
-    throw new Error("indexStats() is not yet supported on the LanceDB cloud");
-  }
-}

nodejs/native.d.ts

@@ -1,208 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-/** A description of an index currently configured on a column */
-export interface IndexConfig {
-  /** The name of the index */
-  name: string
-  /** The type of the index */
-  indexType: string
-  /**
-   * The columns in the index
-   *
-   * Currently this is always an array of size 1. In the future there may
-   * be more columns to represent composite indices.
-   */
-  columns: Array<string>
-}
-/** Statistics about a compaction operation. */
-export interface CompactionStats {
-  /** The number of fragments removed */
-  fragmentsRemoved: number
-  /** The number of new, compacted fragments added */
-  fragmentsAdded: number
-  /** The number of data files removed */
-  filesRemoved: number
-  /** The number of new, compacted data files added */
-  filesAdded: number
-}
-/** Statistics about a cleanup operation */
-export interface RemovalStats {
-  /** The number of bytes removed */
-  bytesRemoved: number
-  /** The number of old versions removed */
-  oldVersionsRemoved: number
-}
-/** Statistics about an optimize operation */
-export interface OptimizeStats {
-  /** Statistics about the compaction operation */
-  compaction: CompactionStats
-  /** Statistics about the removal operation */
-  prune: RemovalStats
-}
-/**
- *  A definition of a column alteration. The alteration changes the column at
- * `path` to have the new name `name`, to be nullable if `nullable` is true,
- * and to have the data type `data_type`. At least one of `rename` or `nullable`
- * must be provided.
- */
-export interface ColumnAlteration {
-  /**
-   * The path to the column to alter. This is a dot-separated path to the column.
-   * If it is a top-level column then it is just the name of the column. If it is
-   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
-   * `c` nested inside a column `b` nested inside a column `a`.
-   */
-  path: string
-  /**
-   * The new name of the column. If not provided then the name will not be changed.
-   * This must be distinct from the names of all other columns in the table.
-   */
-  rename?: string
-  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
-  nullable?: boolean
-}
-/** A definition of a new column to add to a table. */
-export interface AddColumnsSql {
-  /** The name of the new column. */
-  name: string
-  /**
-   * The values to populate the new column with, as a SQL expression.
-   * The expression can reference other columns in the table.
-   */
-  valueSql: string
-}
-export interface IndexStatistics {
-  /** The number of rows indexed by the index */
-  numIndexedRows: number
-  /** The number of rows not indexed */
-  numUnindexedRows: number
-  /** The type of the index */
-  indexType?: string
-  /** The metadata for each index */
-  indices: Array<IndexMetadata>
-}
-export interface IndexMetadata {
-  metricType?: string
-  indexType?: string
-}
-export interface ConnectionOptions {
-  /**
-   * (For LanceDB OSS only): The interval, in seconds, at which to check for
-   * updates to the table from other processes. If None, then consistency is not
-   * checked. For performance reasons, this is the default. For strong
-   * consistency, set this to zero seconds. Then every read will check for
-   * updates from other processes. As a compromise, you can set this to a
-   * non-zero value for eventual consistency. If more than that interval
-   * has passed since the last check, then the table will be checked for updates.
-   * Note: this consistency only applies to read operations. Write operations are
-   * always consistent.
-   */
-  readConsistencyInterval?: number
-  /**
-   * (For LanceDB OSS only): configuration for object storage.
-   *
-   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-   */
-  storageOptions?: Record<string, string>
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  /** Write mode for writing to a table. */
-  mode?: WriteMode
-}
-export interface OpenTableOptions {
-  storageOptions?: Record<string, string>
-}
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string, options: ConnectionOptions): Promise<Connection>
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** List all tables in the dataset. */
-  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class Index {
-  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
-  static btree(): Index
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-/** A builder used to create and run a merge insert operation */
-export class NativeMergeInsertBuilder {
-  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
-  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
-  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
-  execute(buf: Buffer): Promise<void>
-}
-export class Query {
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  nearestTo(vector: Float32Array): VectorQuery
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class VectorQuery {
-  column(column: string): void
-  distanceType(distanceType: string): void
-  postfilter(): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  bypassVectorIndex(): void
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class Table {
-  name: string
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Promise<Buffer>
-  add(buf: Buffer, mode: string): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<number>
-  delete(predicate: string): Promise<void>
-  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
-  query(): Query
-  vectorSearch(vector: Float32Array): VectorQuery
-  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
-  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
-  dropColumns(columns: Array<string>): Promise<void>
-  version(): Promise<number>
-  checkout(version: number): Promise<void>
-  checkoutLatest(): Promise<void>
-  restore(): Promise<void>
-  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
-  listIndices(): Promise<Array<IndexConfig>>
-  indexStats(indexName: string): Promise<IndexStatistics | null>
-  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
-}

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.10.0-beta.1",
+	"version": "0.11.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.10.0-beta.1",
+	"version": "0.11.0-beta.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.10.0-beta.1",
+	"version": "0.11.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.10.0-beta.1",
+	"version": "0.11.0-beta.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.10.0-beta.1",
+	"version": "0.11.0-beta.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.10.0-beta.1",
+  "version": "0.11.0-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.10.0-beta.1",
+      "version": "0.11.0-beta.1",
       "cpu": [
         "x64",
         "arm64"
@@ -18,7 +18,6 @@
         "win32"
       ],
       "dependencies": {
-        "axios": "^1.7.2",
         "reflect-metadata": "^0.2.2"
       },
       "devDependencies": {
@@ -30,6 +29,7 @@
         "@napi-rs/cli": "^2.18.3",
         "@types/axios": "^0.14.0",
         "@types/jest": "^29.1.2",
+        "@types/node": "^22.7.4",
         "@types/tmp": "^0.2.6",
         "apache-arrow-13": "npm:apache-arrow@13.0.0",
         "apache-arrow-14": "npm:apache-arrow@14.0.0",
@@ -4648,11 +4648,12 @@
       "optional": true
     },
     "node_modules/@types/node": {
-      "version": "20.14.11",
+      "version": "22.7.4",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.4.tgz",
-      "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==",
+      "integrity": "sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==",
+      "devOptional": true,
       "dependencies": {
-        "undici-types": "~5.26.4"
+        "undici-types": "~6.19.2"
       }
     },
     "node_modules/@types/node-fetch": {
@@ -4665,6 +4666,12 @@
         "form-data": "^4.0.0"
       }
     },
+    "node_modules/@types/node/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "devOptional": true
+    },
     "node_modules/@types/pad-left": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
@@ -4963,6 +4970,21 @@
         "arrow2csv": "bin/arrow2csv.cjs"
       }
     },
+    "node_modules/apache-arrow-15/node_modules/@types/node": {
+      "version": "20.16.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.10.tgz",
+      "integrity": "sha512-vQUKgWTjEIRFCvK6CyriPH3MZYiYlNy0fKiEYHWbcoWLEgs4opurGGKlebrTLqdSMIbXImH6XExNiIyNUv3WpA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
+    "node_modules/apache-arrow-15/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true
+    },
     "node_modules/apache-arrow-16": {
       "name": "apache-arrow",
       "version": "16.0.0",
@@ -4984,6 +5006,21 @@
         "arrow2csv": "bin/arrow2csv.cjs"
       }
     },
+    "node_modules/apache-arrow-16/node_modules/@types/node": {
+      "version": "20.16.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.10.tgz",
+      "integrity": "sha512-vQUKgWTjEIRFCvK6CyriPH3MZYiYlNy0fKiEYHWbcoWLEgs4opurGGKlebrTLqdSMIbXImH6XExNiIyNUv3WpA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
+    "node_modules/apache-arrow-16/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true
+    },
     "node_modules/apache-arrow-17": {
       "name": "apache-arrow",
       "version": "17.0.0",
@@ -5011,12 +5048,42 @@
       "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==",
       "dev": true
     },
+    "node_modules/apache-arrow-17/node_modules/@types/node": {
+      "version": "20.16.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.10.tgz",
+      "integrity": "sha512-vQUKgWTjEIRFCvK6CyriPH3MZYiYlNy0fKiEYHWbcoWLEgs4opurGGKlebrTLqdSMIbXImH6XExNiIyNUv3WpA==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
     "node_modules/apache-arrow-17/node_modules/flatbuffers": {
       "version": "24.3.25",
       "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.3.25.tgz",
       "integrity": "sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ==",
       "dev": true
     },
+    "node_modules/apache-arrow-17/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "dev": true
+    },
+    "node_modules/apache-arrow/node_modules/@types/node": {
+      "version": "20.16.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.10.tgz",
+      "integrity": "sha512-vQUKgWTjEIRFCvK6CyriPH3MZYiYlNy0fKiEYHWbcoWLEgs4opurGGKlebrTLqdSMIbXImH6XExNiIyNUv3WpA==",
+      "peer": true,
+      "dependencies": {
+        "undici-types": "~6.19.2"
+      }
+    },
+    "node_modules/apache-arrow/node_modules/undici-types": {
+      "version": "6.19.8",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
+      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "peer": true
+    },
     "node_modules/argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -5046,12 +5113,14 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "devOptional": true
     },
     "node_modules/axios": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz",
       "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==",
+      "dev": true,
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
@@ -5536,6 +5605,7 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "devOptional": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -5723,6 +5793,7 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "devOptional": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -6248,6 +6319,7 @@
       "version": "1.15.6",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
       "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
+      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -6267,6 +6339,7 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "devOptional": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -7773,6 +7846,7 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "devOptional": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -7781,6 +7855,7 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "devOptional": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -8393,7 +8468,8 @@
     "node_modules/proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
+      "dev": true
     },
     "node_modules/pump": {
       "version": "3.0.0",
@@ -9561,7 +9637,8 @@
     "node_modules/undici-types": {
       "version": "5.26.5",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
-      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "optional": true
     },
     "node_modules/update-browserslist-db": {
       "version": "1.0.13",

nodejs/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0-beta.1",
+  "version": "0.11.0-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -40,6 +40,7 @@
     "@napi-rs/cli": "^2.18.3",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
+    "@types/node": "^22.7.4",
     "@types/tmp": "^0.2.6",
     "apache-arrow-13": "npm:apache-arrow@13.0.0",
     "apache-arrow-14": "npm:apache-arrow@14.0.0",
@@ -66,8 +67,8 @@
   "os": ["darwin", "linux", "win32"],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
+    "build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
-    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
+    "build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
     "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts && shx cp lancedb/*.node dist/",
     "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
@@ -81,7 +82,6 @@
     "version": "napi version"
   },
   "dependencies": {
-    "axios": "^1.7.2",
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {

nodejs/src/connection.rs

@@ -68,6 +68,24 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
+
+        let client_config = options.client_config.unwrap_or_default();
+        builder = builder.client_config(client_config.into());
+
+        if let Some(api_key) = options.api_key {
+            builder = builder.api_key(&api_key);
+        }
+
+        if let Some(region) = options.region {
+            builder = builder.region(&region);
+        } else {
+            builder = builder.region("us-east-1");
+        }
+
+        if let Some(host_override) = options.host_override {
+            builder = builder.host_override(&host_override);
+        }
+
         Ok(Self::inner_new(
             builder
                 .execute()
@@ -130,6 +148,7 @@ impl Connection {
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
         let mode = Self::parse_create_mode_str(&mode)?;
         let mut builder = self.get_inner()?.create_table(&name, batches).mode(mode);
+
         if let Some(storage_options) = storage_options {
             for (key, value) in storage_options {
                 builder = builder.storage_option(key, value);

nodejs/src/lib.rs

@@ -22,6 +22,7 @@ mod index;
 mod iterator;
 pub mod merge;
 mod query;
+pub mod remote;
 mod table;
 mod util;
 
@@ -42,6 +43,19 @@ pub struct ConnectionOptions {
     ///
     /// The available options are described at https://lancedb.github.io/lancedb/guides/storage/
     pub storage_options: Option<HashMap<String, String>>,
+
+    /// (For LanceDB cloud only): configuration for the remote HTTP client.
+    pub client_config: Option<remote::ClientConfig>,
+    /// (For LanceDB cloud only): the API key to use with LanceDB Cloud.
+    ///
+    /// Can also be set via the environment variable `LANCEDB_API_KEY`.
+    pub api_key: Option<String>,
+    /// (For LanceDB cloud only): the region to use for LanceDB cloud.
+    /// Defaults to 'us-east-1'.
+    pub region: Option<String>,
+    /// (For LanceDB cloud only): the host to use for LanceDB cloud. Used
+    /// for testing purposes.
+    pub host_override: Option<String>,
 }
 
 /// Write mode for writing a table.

nodejs/src/remote.rs

@@ -0,0 +1,120 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use napi_derive::*;
+
+/// Timeout configuration for remote HTTP client.
+#[napi(object)]
+#[derive(Debug)]
+pub struct TimeoutConfig {
+    /// The timeout for establishing a connection in seconds. Default is 120
+    /// seconds (2 minutes). This can also be set via the environment variable
+    /// `LANCE_CLIENT_CONNECT_TIMEOUT`, as an integer number of seconds.
+    pub connect_timeout: Option<f64>,
+    /// The timeout for reading data from the server in seconds. Default is 300
+    /// seconds (5 minutes). This can also be set via the environment variable
+    /// `LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
+    pub read_timeout: Option<f64>,
+    /// The timeout for keeping idle connections in the connection pool in seconds.
+    /// Default is 300 seconds (5 minutes). This can also be set via the
+    /// environment variable `LANCE_CLIENT_CONNECTION_TIMEOUT`, as an integer
+    /// number of seconds.
+    pub pool_idle_timeout: Option<f64>,
+}
+
+/// Retry configuration for the remote HTTP client.
+#[napi(object)]
+#[derive(Debug)]
+pub struct RetryConfig {
+    /// The maximum number of retries for a request. Default is 3. You can also
+    /// set this via the environment variable `LANCE_CLIENT_MAX_RETRIES`.
+    pub retries: Option<u8>,
+    /// The maximum number of retries for connection errors. Default is 3. You
+    /// can also set this via the environment variable `LANCE_CLIENT_CONNECT_RETRIES`.
+    pub connect_retries: Option<u8>,
+    /// The maximum number of retries for read errors. Default is 3. You can also
+    /// set this via the environment variable `LANCE_CLIENT_READ_RETRIES`.
+    pub read_retries: Option<u8>,
+    /// The backoff factor to apply between retries. Default is 0.25. Between each retry
+    /// the client will wait for the amount of seconds:
+    /// `{backoff factor} * (2 ** ({number of previous retries}))`. So for the default
+    /// of 0.25, the first retry will wait 0.25 seconds, the second retry will wait 0.5
+    /// seconds, the third retry will wait 1 second, etc.
+    ///
+    /// You can also set this via the environment variable
+    /// `LANCE_CLIENT_RETRY_BACKOFF_FACTOR`.
+    pub backoff_factor: Option<f64>,
+    /// The jitter to apply to the backoff factor, in seconds. Default is 0.25.
+    ///
+    /// A random value between 0 and `backoff_jitter` will be added to the backoff
+    /// factor in seconds. So for the default of 0.25 seconds, between 0 and 250
+    /// milliseconds will be added to the sleep between each retry.
+    ///
+    /// You can also set this via the environment variable
+    /// `LANCE_CLIENT_RETRY_BACKOFF_JITTER`.
+    pub backoff_jitter: Option<f64>,
+    /// The HTTP status codes for which to retry the request. Default is
+    /// [429, 500, 502, 503].
+    ///
+    /// You can also set this via the environment variable
+    /// `LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.
+    pub statuses: Option<Vec<u16>>,
+}
+
+#[napi(object)]
+#[derive(Debug, Default)]
+pub struct ClientConfig {
+    pub user_agent: Option<String>,
+    pub retry_config: Option<RetryConfig>,
+    pub timeout_config: Option<TimeoutConfig>,
+}
+
+impl From<TimeoutConfig> for lancedb::remote::TimeoutConfig {
+    fn from(config: TimeoutConfig) -> Self {
+        Self {
+            connect_timeout: config
+                .connect_timeout
+                .map(std::time::Duration::from_secs_f64),
+            read_timeout: config.read_timeout.map(std::time::Duration::from_secs_f64),
+            pool_idle_timeout: config
+                .pool_idle_timeout
+                .map(std::time::Duration::from_secs_f64),
+        }
+    }
+}
+
+impl From<RetryConfig> for lancedb::remote::RetryConfig {
+    fn from(config: RetryConfig) -> Self {
+        Self {
+            retries: config.retries,
+            connect_retries: config.connect_retries,
+            read_retries: config.read_retries,
+            backoff_factor: config.backoff_factor.map(|v| v as f32),
+            backoff_jitter: config.backoff_jitter.map(|v| v as f32),
+            statuses: config.statuses,
+        }
+    }
+}
+
+impl From<ClientConfig> for lancedb::remote::ClientConfig {
+    fn from(config: ClientConfig) -> Self {
+        Self {
+            user_agent: config
+                .user_agent
+                .unwrap_or(concat!("LanceDB-Node-Client/", env!("CARGO_PKG_VERSION")).to_string()),
+            retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
+            timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
+        }
+    }
+}

nodejs/src/table.rs

@@ -156,7 +156,7 @@ impl Table {
         &self,
         only_if: Option<String>,
         columns: Vec<(String, String)>,
-    ) -> napi::Result<()> {
+    ) -> napi::Result<u64> {
         let mut op = self.inner_ref()?.update();
         if let Some(only_if) = only_if {
             op = op.only_if(only_if);
@@ -337,7 +337,7 @@ impl Table {
 
     #[napi(catch_unwind)]
     pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
-        let tbl = self.inner_ref()?.as_native().unwrap();
+        let tbl = self.inner_ref()?;
         let stats = tbl.index_stats(&index_name).await.default_error()?;
         Ok(stats.map(IndexStatistics::from))
     }
@@ -480,32 +480,22 @@ pub struct IndexStatistics {
     /// The number of rows not indexed
     pub num_unindexed_rows: f64,
     /// The type of the index
-    pub index_type: Option<String>,
+    pub index_type: String,
-    /// The metadata for each index
+    /// The type of the distance function used by the index. This is only
-    pub indices: Vec<IndexMetadata>,
+    /// present for vector indices. Scalar and full text search indices do
+    /// not have a distance function.
+    pub distance_type: Option<String>,
+    /// The number of parts this index is split into.
+    pub num_indices: Option<u32>,
 }
 impl From<lancedb::index::IndexStatistics> for IndexStatistics {
     fn from(value: lancedb::index::IndexStatistics) -> Self {
         Self {
             num_indexed_rows: value.num_indexed_rows as f64,
             num_unindexed_rows: value.num_unindexed_rows as f64,
-            index_type: value.index_type.map(|t| format!("{:?}", t)),
+            index_type: value.index_type.to_string(),
-            indices: value.indices.into_iter().map(Into::into).collect(),
+            distance_type: value.distance_type.map(|d| d.to_string()),
-        }
+            num_indices: value.num_indices,
-    }
-}
-
-#[napi(object)]
-pub struct IndexMetadata {
-    pub metric_type: Option<String>,
-    pub index_type: Option<String>,
-}
-
-impl From<lancedb::index::IndexMetadata> for IndexMetadata {
-    fn from(value: lancedb::index::IndexMetadata) -> Self {
-        Self {
-            metric_type: value.metric_type,
-            index_type: value.index_type,
         }
     }
 }

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.13.0"
+current_version = "0.14.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.13.0"
+version = "0.14.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
@@ -22,8 +22,6 @@ pyo3 = { version = "0.21", features = ["extension-module", "abi3-py38", "gil-ref
 # pyo3-asyncio = { version = "0.20", features = ["attributes", "tokio-runtime"] }
 pyo3-asyncio-0-21 = { version = "0.21.0", features = ["attributes", "tokio-runtime"] }
 
-# Prevent dynamic linking of lzma, which comes from datafusion
-lzma-sys = { version = "*", features = ["static"] }
 pin-project = "1.1.5"
 futures.workspace = true
 tokio = { version = "1.36.0", features = ["sync"] }
@@ -35,4 +33,6 @@ pyo3-build-config = { version = "0.20.3", features = [
 ] }
 
 [features]
+default = ["remote"]
 fp16kernels = ["lancedb/fp16kernels"]
+remote = ["lancedb/remote"]

python/pyproject.toml

@@ -3,7 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "pylance==0.17.0",
+    "pylance==0.18.2",
     "requests>=2.31.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",

python/python/lancedb/__init__.py

@@ -19,6 +19,8 @@ from typing import Dict, Optional, Union, Any
 
 __version__ = importlib.metadata.version("lancedb")
 
+from lancedb.remote import ClientConfig
+
 from ._lancedb import connect as lancedb_connect
 from .common import URI, sanitize_uri
 from .db import AsyncConnection, DBConnection, LanceDBConnection
@@ -120,7 +122,7 @@ async def connect_async(
     region: str = "us-east-1",
     host_override: Optional[str] = None,
     read_consistency_interval: Optional[timedelta] = None,
-    request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
+    client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
     storage_options: Optional[Dict[str, str]] = None,
 ) -> AsyncConnection:
     """Connect to a LanceDB database.
@@ -148,6 +150,10 @@ async def connect_async(
         the last check, then the table will be checked for updates. Note: this
         consistency only applies to read operations. Write operations are
         always consistent.
+    client_config: ClientConfig or dict, optional
+        Configuration options for the LanceDB Cloud HTTP client. If a dict, then
+        the keys are the attributes of the ClientConfig class. If None, then the
+        default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
         https://lancedb.github.io/lancedb/guides/storage/
@@ -160,7 +166,13 @@ async def connect_async(
     ...     # For a local directory, provide a path to the database
     ...     db = await lancedb.connect_async("~/.lancedb")
     ...     # For object storage, use a URI prefix
-    ...     db = await lancedb.connect_async("s3://my-bucket/lancedb")
+    ...     db = await lancedb.connect_async("s3://my-bucket/lancedb",
+    ...                                      storage_options={
+    ...                                          "aws_access_key_id": "***"})
+    ...     # Connect to LanceDB cloud
+    ...     db = await lancedb.connect_async("db://my_database", api_key="ldb_...",
+    ...                                      client_config={
+    ...                                          "retry_config": {"retries": 5}})
 
     Returns
     -------
@@ -172,6 +184,9 @@ async def connect_async(
     else:
         read_consistency_interval_secs = None
 
+    if isinstance(client_config, dict):
+        client_config = ClientConfig(**client_config)
+
     return AsyncConnection(
         await lancedb_connect(
             sanitize_uri(uri),
@@ -179,6 +194,7 @@ async def connect_async(
             region,
             host_override,
             read_consistency_interval_secs,
+            client_config,
             storage_options,
         )
     )

python/python/lancedb/common.py

@@ -20,7 +20,7 @@ from .util import safe_import_pandas
 
 pd = safe_import_pandas()
 
-DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
+DATA = Union[List[dict], "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
 VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
 URI = Union[str, Path]
 VECTOR_COLUMN_NAME = "vector"

python/python/lancedb/db.py

@@ -96,7 +96,7 @@ class DBConnection(EnforceOverrides):
             User must provide at least one of `data` or `schema`.
             Acceptable types are:
 
-            - dict or list-of-dict
+            - list-of-dict
 
             - pandas.DataFrame
 
@@ -579,7 +579,7 @@ class AsyncConnection(object):
             User must provide at least one of `data` or `schema`.
             Acceptable types are:
 
-            - dict or list-of-dict
+            - list-of-dict
 
             - pandas.DataFrame
 
@@ -610,14 +610,13 @@ class AsyncConnection(object):
             connection will be inherited by the table, but can be overridden here.
             See available options at
             https://lancedb.github.io/lancedb/guides/storage/
-        data_storage_version: optional, str, default "legacy"
+        data_storage_version: optional, str, default "stable"
             The version of the data storage format to use. Newer versions are more
             efficient but require newer versions of lance to read.  The default is
-            "legacy" which will use the legacy v1 version.  See the user guide
+            "stable" which will use the legacy v2 version.  See the user guide
             for more details.
-        use_legacy_format: bool, optional, default True. (Deprecated)
+        use_legacy_format: bool, optional, default False. (Deprecated)
             If True, use the legacy format for the table. If False, use the new format.
-            The default is True while the new format is in beta.
             This method is deprecated, use `data_storage_version` instead.
         enable_v2_manifest_paths: bool, optional, default False
             Use the new V2 manifest paths. These paths provide more efficient
@@ -759,9 +758,7 @@ class AsyncConnection(object):
             mode = "exist_ok"
 
         if not data_storage_version:
-            data_storage_version = (
+            data_storage_version = "legacy" if use_legacy_format else "stable"
-                "legacy" if use_legacy_format is None or use_legacy_format else "stable"
-            )
 
         if data is None:
             new_table = await self._inner.create_empty_table(

python/python/lancedb/dependencies.py

@@ -0,0 +1,259 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+#
+# The following code is originally from https://github.com/pola-rs/polars/blob/ea4389c31b0e87ddf20a85e4c3797b285966edb6/py-polars/polars/dependencies.py
+# and is licensed under the MIT license:
+#
+# License: MIT, Copyright (c) 2020 Ritchie Vink
+# https://github.com/pola-rs/polars/blob/main/LICENSE
+#
+# It has been modified by the LanceDB developers
+# to fit the needs of the LanceDB project.
+
+
+from __future__ import annotations
+
+import re
+import sys
+from functools import lru_cache
+from importlib import import_module
+from importlib.util import find_spec
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, ClassVar, Hashable, cast
+
+_NUMPY_AVAILABLE = True
+_PANDAS_AVAILABLE = True
+_POLARS_AVAILABLE = True
+_TORCH_AVAILABLE = True
+_HUGGING_FACE_AVAILABLE = True
+_TENSORFLOW_AVAILABLE = True
+_RAY_AVAILABLE = True
+
+
+class _LazyModule(ModuleType):
+    """
+    Module that can act both as a lazy-loader and as a proxy.
+
+    Notes
+    -----
+    We do NOT register this module with `sys.modules` so as not to cause
+    confusion in the global environment. This way we have a valid proxy
+    module for our own use, but it lives _exclusively_ within lance.
+
+    """
+
+    __lazy__ = True
+
+    _mod_pfx: ClassVar[dict[str, str]] = {
+        "numpy": "np.",
+        "pandas": "pd.",
+        "polars": "pl.",
+        "torch": "torch.",
+        "tensorflow": "tf.",
+        "ray": "ray.",
+    }
+
+    def __init__(
+        self,
+        module_name: str,
+        *,
+        module_available: bool,
+    ) -> None:
+        """
+        Initialise lazy-loading proxy module.
+
+        Parameters
+        ----------
+        module_name : str
+            the name of the module to lazy-load (if available).
+
+        module_available : bool
+            indicate if the referenced module is actually available (we will proxy it
+            in both cases, but raise a helpful error when invoked if it doesn't exist).
+
+        """
+        self._module_available = module_available
+        self._module_name = module_name
+        self._globals = globals()
+        super().__init__(module_name)
+
+    def _import(self) -> ModuleType:
+        # import the referenced module, replacing the proxy in this module's globals
+        module = import_module(self.__name__)
+        self._globals[self._module_name] = module
+        self.__dict__.update(module.__dict__)
+        return module
+
+    def __getattr__(self, attr: Any) -> Any:
+        # have "hasattr('__wrapped__')" return False without triggering import
+        # (it's for decorators, not modules, but keeps "make doctest" happy)
+        if attr == "__wrapped__":
+            raise AttributeError(
+                f"{self._module_name!r} object has no attribute {attr!r}"
+            )
+
+        # accessing the proxy module's attributes triggers import of the real thing
+        if self._module_available:
+            # import the module and return the requested attribute
+            module = self._import()
+            return getattr(module, attr)
+
+        # user has not installed the proxied/lazy module
+        elif attr == "__name__":
+            return self._module_name
+        elif re.match(r"^__\w+__$", attr) and attr != "__version__":
+            # allow some minimal introspection on private module
+            # attrs to avoid unnecessary error-handling elsewhere
+            return None
+        else:
+            # all other attribute access raises a helpful exception
+            pfx = self._mod_pfx.get(self._module_name, "")
+            raise ModuleNotFoundError(
+                f"{pfx}{attr} requires {self._module_name!r} module to be installed"
+            ) from None
+
+
+def _lazy_import(module_name: str) -> tuple[ModuleType, bool]:
+    """
+    Lazy import the given module; avoids up-front import costs.
+
+    Parameters
+    ----------
+    module_name : str
+        name of the module to import, eg: "polars".
+
+    Notes
+    -----
+    If the requested module is not available (eg: has not been installed), a proxy
+    module is created in its place, which raises an exception on any attribute
+    access. This allows for import and use as normal, without requiring explicit
+    guard conditions - if the module is never used, no exception occurs; if it
+    is, then a helpful exception is raised.
+
+    Returns
+    -------
+    tuple of (Module, bool)
+        A lazy-loading module and a boolean indicating if the requested/underlying
+        module exists (if not, the returned module is a proxy).
+
+    """
+    # check if module is LOADED
+    if module_name in sys.modules:
+        return sys.modules[module_name], True
+
+    # check if module is AVAILABLE
+    try:
+        module_spec = find_spec(module_name)
+        module_available = not (module_spec is None or module_spec.loader is None)
+    except ModuleNotFoundError:
+        module_available = False
+
+    # create lazy/proxy module that imports the real one on first use
+    # (or raises an explanatory ModuleNotFoundError if not available)
+    return (
+        _LazyModule(
+            module_name=module_name,
+            module_available=module_available,
+        ),
+        module_available,
+    )
+
+
+if TYPE_CHECKING:
+    import datasets
+    import numpy
+    import pandas
+    import polars
+    import ray
+    import tensorflow
+    import torch
+else:
+    # heavy/optional third party libs
+    numpy, _NUMPY_AVAILABLE = _lazy_import("numpy")
+    pandas, _PANDAS_AVAILABLE = _lazy_import("pandas")
+    polars, _POLARS_AVAILABLE = _lazy_import("polars")
+    torch, _TORCH_AVAILABLE = _lazy_import("torch")
+    datasets, _HUGGING_FACE_AVAILABLE = _lazy_import("datasets")
+    tensorflow, _TENSORFLOW_AVAILABLE = _lazy_import("tensorflow")
+    ray, _RAY_AVAILABLE = _lazy_import("ray")
+
+
+@lru_cache(maxsize=None)
+def _might_be(cls: type, type_: str) -> bool:
+    # infer whether the given class "might" be associated with the given
+    # module (in which case it's reasonable to do a real isinstance check)
+    try:
+        return any(f"{type_}." in str(o) for o in cls.mro())
+    except TypeError:
+        return False
+
+
+def _check_for_numpy(obj: Any, *, check_type: bool = True) -> bool:
+    return _NUMPY_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "numpy"
+    )
+
+
+def _check_for_pandas(obj: Any, *, check_type: bool = True) -> bool:
+    return _PANDAS_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "pandas"
+    )
+
+
+def _check_for_polars(obj: Any, *, check_type: bool = True) -> bool:
+    return _POLARS_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "polars"
+    )
+
+
+def _check_for_torch(obj: Any, *, check_type: bool = True) -> bool:
+    return _TORCH_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "torch"
+    )
+
+
+def _check_for_hugging_face(obj: Any, *, check_type: bool = True) -> bool:
+    return _HUGGING_FACE_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "datasets"
+    )
+
+
+def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool:
+    return _TENSORFLOW_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "tensorflow"
+    )
+
+
+def _check_for_ray(obj: Any, *, check_type: bool = True) -> bool:
+    return _RAY_AVAILABLE and _might_be(
+        cast(Hashable, type(obj) if check_type else obj), "ray"
+    )
+
+
+__all__ = [
+    # lazy-load third party libs
+    "datasets",
+    "numpy",
+    "pandas",
+    "polars",
+    "ray",
+    "tensorflow",
+    "torch",
+    # lazy utilities
+    "_check_for_hugging_face",
+    "_check_for_numpy",
+    "_check_for_pandas",
+    "_check_for_polars",
+    "_check_for_tensorflow",
+    "_check_for_torch",
+    "_check_for_ray",
+    "_LazyModule",
+    # exported flags/guards
+    "_NUMPY_AVAILABLE",
+    "_PANDAS_AVAILABLE",
+    "_POLARS_AVAILABLE",
+    "_TORCH_AVAILABLE",
+    "_HUGGING_FACE_AVAILABLE",
+    "_TENSORFLOW_AVAILABLE",
+    "_RAY_AVAILABLE",
+]

python/python/lancedb/embeddings/base.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from abc import ABC, abstractmethod
 from typing import List, Union
 
@@ -34,7 +25,7 @@ class EmbeddingFunction(BaseModel, ABC):
 
     __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
     max_retries: int = (
-        7  # Setitng 0 disables retires. Maybe this should not be enabled by default,
+        7  # Setting 0 disables retires. Maybe this should not be enabled by default,
     )
     _ndims: int = PrivateAttr()
 
@@ -46,22 +37,37 @@ class EmbeddingFunction(BaseModel, ABC):
         return cls(**kwargs)
 
     @abstractmethod
-    def compute_query_embeddings(self, *args, **kwargs) -> List[np.array]:
+    def compute_query_embeddings(self, *args, **kwargs) -> list[Union[np.array, None]]:
         """
         Compute the embeddings for a given user query
+
+        Returns
+        -------
+        A list of embeddings for each input. The embedding of each input can be None
+        when the embedding is not valid.
         """
         pass
 
     @abstractmethod
-    def compute_source_embeddings(self, *args, **kwargs) -> List[np.array]:
+    def compute_source_embeddings(self, *args, **kwargs) -> list[Union[np.array, None]]:
-        """
+        """Compute the embeddings for the source column in the database
-        Compute the embeddings for the source column in the database
+
+        Returns
+        -------
+        A list of embeddings for each input. The embedding of each input can be None
+        when the embedding is not valid.
         """
         pass
 
-    def compute_query_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+    def compute_query_embeddings_with_retry(
-        """
+        self, *args, **kwargs
-        Compute the embeddings for a given user query with retries
+    ) -> list[Union[np.array, None]]:
+        """Compute the embeddings for a given user query with retries
+
+        Returns
+        -------
+        A list of embeddings for each input. The embedding of each input can be None
+        when the embedding is not valid.
         """
         return retry_with_exponential_backoff(
             self.compute_query_embeddings, max_retries=self.max_retries
@@ -70,9 +76,15 @@ class EmbeddingFunction(BaseModel, ABC):
             **kwargs,
         )
 
-    def compute_source_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+    def compute_source_embeddings_with_retry(
-        """
+        self, *args, **kwargs
-        Compute the embeddings for the source column in the database with retries
+    ) -> list[Union[np.array, None]]:
+        """Compute the embeddings for the source column in the database with retries.
+
+        Returns
+        -------
+        A list of embeddings for each input. The embedding of each input can be None
+        when the embedding is not valid.
         """
         return retry_with_exponential_backoff(
             self.compute_source_embeddings, max_retries=self.max_retries
@@ -94,8 +106,14 @@ class EmbeddingFunction(BaseModel, ABC):
         from ..pydantic import PYDANTIC_VERSION
 
         if PYDANTIC_VERSION.major < 2:
-            return dict(self)
+            return {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
-        return self.model_dump()
+        return self.model_dump(
+            exclude={
+                field_name
+                for field_name in self.model_fields
+                if field_name.startswith("_")
+            }
+        )
 
     @abstractmethod
     def ndims(self):
@@ -144,18 +162,20 @@ class TextEmbeddingFunction(EmbeddingFunction):
     A callable ABC for embedding functions that take text as input
     """
 
-    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+    def compute_query_embeddings(
+        self, query: str, *args, **kwargs
+    ) -> list[Union[np.array, None]]:
         return self.compute_source_embeddings(query, *args, **kwargs)
 
-    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+    def compute_source_embeddings(
+        self, texts: TEXT, *args, **kwargs
+    ) -> list[Union[np.array, None]]:
         texts = self.sanitize_input(texts)
         return self.generate_embeddings(texts)
 
     @abstractmethod
     def generate_embeddings(
         self, texts: Union[List[str], np.ndarray], *args, **kwargs
-    ) -> List[np.array]:
+    ) -> list[Union[np.array, None]]:
-        """
+        """Generate the embeddings for the given texts"""
-        Generate the embeddings for the given texts
-        """
         pass

python/python/lancedb/embeddings/ollama.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from functools import cached_property
 from typing import TYPE_CHECKING, List, Optional, Union
 
@@ -19,6 +10,7 @@ from .registry import register
 
 if TYPE_CHECKING:
     import numpy as np
+    import ollama
 
 
 @register("ollama")
@@ -39,17 +31,20 @@ class OllamaEmbeddings(TextEmbeddingFunction):
     def ndims(self):
         return len(self.generate_embeddings(["foo"])[0])
 
-    def _compute_embedding(self, text):
+    def _compute_embedding(self, text) -> Union["np.array", None]:
-        return self._ollama_client.embeddings(
+        return (
-            model=self.name,
+            self._ollama_client.embeddings(
-            prompt=text,
+                model=self.name,
-            options=self.options,
+                prompt=text,
-            keep_alive=self.keep_alive,
+                options=self.options,
-        )["embedding"]
+                keep_alive=self.keep_alive,
+            )["embedding"]
+            or None
+        )
 
     def generate_embeddings(
         self, texts: Union[List[str], "np.ndarray"]
-    ) -> List["np.array"]:
+    ) -> list[Union["np.array", None]]:
         """
         Get the embeddings for the given texts
 
@@ -63,7 +58,7 @@ class OllamaEmbeddings(TextEmbeddingFunction):
         return embeddings
 
     @cached_property
-    def _ollama_client(self):
+    def _ollama_client(self) -> "ollama.Client":
         ollama = attempt_import_or_raise("ollama")
         # ToDo explore ollama.AsyncClient
         return ollama.Client(host=self.host, **self.ollama_client_kwargs)

python/python/lancedb/embeddings/openai.py

@@ -1,17 +1,9 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from functools import cached_property
 from typing import TYPE_CHECKING, List, Optional, Union
+import logging
 
 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
@@ -89,17 +81,26 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
         texts: list[str] or np.ndarray (of str)
             The texts to embed
         """
+        openai = attempt_import_or_raise("openai")
+
         # TODO retry, rate limit, token limit
-        if self.name == "text-embedding-ada-002":
+        try:
-            rs = self._openai_client.embeddings.create(input=texts, model=self.name)
+            if self.name == "text-embedding-ada-002":
-        else:
+                rs = self._openai_client.embeddings.create(input=texts, model=self.name)
-            kwargs = {
+            else:
-                "input": texts,
+                kwargs = {
-                "model": self.name,
+                    "input": texts,
-            }
+                    "model": self.name,
-            if self.dim:
+                }
-                kwargs["dimensions"] = self.dim
+                if self.dim:
-            rs = self._openai_client.embeddings.create(**kwargs)
+                    kwargs["dimensions"] = self.dim
+                rs = self._openai_client.embeddings.create(**kwargs)
+        except openai.BadRequestError:
+            logging.exception("Bad request: %s", texts)
+            return [None] * len(texts)
+        except Exception:
+            logging.exception("OpenAI embeddings error")
+            raise
         return [v.embedding for v in rs.data]
 
     @cached_property

python/python/lancedb/embeddings/transformers.py

@@ -40,6 +40,11 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         The device to use for the model. Default is "cpu".
     show_progress_bar : bool
         Whether to show a progress bar when loading the model. Default is True.
+    trust_remote_code : bool
+        Whether or not to allow for custom models defined on the HuggingFace
+        Hub in their own modeling files. This option should only be set to True
+        for repositories you trust and in which you have read the code, as it
+        will execute code present on the Hub on your local machine.
 
     to download package, run :
         `pip install transformers`
@@ -49,6 +54,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
 
     name: str = "colbert-ir/colbertv2.0"
     device: str = "cpu"
+    trust_remote_code: bool = False
     _tokenizer: Any = PrivateAttr()
     _model: Any = PrivateAttr()
 
@@ -57,7 +63,9 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         self._ndims = None
         transformers = attempt_import_or_raise("transformers")
         self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.name)
-        self._model = transformers.AutoModel.from_pretrained(self.name)
+        self._model = transformers.AutoModel.from_pretrained(
+            self.name, trust_remote_code=self.trust_remote_code
+        )
         self._model.to(self.device)
 
     if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat

python/python/lancedb/index.py

@@ -83,7 +83,108 @@ class FTS:
 
 
 class HnswPq:
-    """Describe a Hnswpq index configuration."""
+    """Describe a HNSW-PQ index configuration.
+
+    HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+    It is a variant of the HNSW algorithm that uses product quantization to compress
+    the vectors. To create an HNSW-PQ index, you can specify the following parameters:
+
+    Parameters
+    ----------
+
+    distance_type: str, default "L2"
+
+        The distance metric used to train the index.
+
+        The following distance types are available:
+
+        "l2" - Euclidean distance. This is a very common distance metric that
+        accounts for both magnitude and direction when determining the distance
+        between vectors. L2 distance has a range of [0, ∞).
+
+        "cosine" - Cosine distance.  Cosine distance is a distance metric
+        calculated from the cosine similarity between two vectors. Cosine
+        similarity is a measure of similarity between two non-zero vectors of an
+        inner product space. It is defined to equal the cosine of the angle
+        between them.  Unlike L2, the cosine distance is not affected by the
+        magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+        L2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+    num_partitions, default sqrt(num_rows)
+
+        The number of IVF partitions to create.
+
+        For HNSW, we recommend a small number of partitions. Setting this to 1 works
+        well for most tables. For very large tables, training just one HNSW graph
+        will require too much memory. Each partition becomes its own HNSW graph, so
+        setting this value higher reduces the peak memory use of training.
+
+    num_sub_vectors, default is vector dimension / 16
+
+        Number of sub-vectors of PQ.
+
+        This value controls how much the vector is compressed during the
+        quantization step. The more sub vectors there are the less the vector is
+        compressed.  The default is the dimension of the vector divided by 16.
+        If the dimension is not evenly divisible by 16 we use the dimension
+        divided by 8.
+
+        The above two cases are highly preferred.  Having 8 or 16 values per
+        subvector allows us to use efficient SIMD instructions.
+
+        If the dimension is not visible by 8 then we use 1 subvector.  This is not
+        ideal and will likely result in poor performance.
+
+    max_iterations, default 50
+
+        Max iterations to train kmeans.
+
+        When training an IVF index we use kmeans to calculate the partitions.  This
+        parameter controls how many iterations of kmeans to run.
+
+        Increasing this might improve the quality of the index but in most cases the
+        parameter is unused because kmeans will converge with fewer iterations.  The
+        parameter is only used in cases where kmeans does not appear to converge.  In
+        those cases it is unlikely that setting this larger will lead to the index
+        converging anyways.
+
+    sample_rate, default 256
+
+        The rate used to calculate the number of training vectors for kmeans.
+
+        When an IVF index is trained, we need to calculate partitions.  These are
+        groups of vectors that are similar to each other.  To do this we use an
+        algorithm called kmeans.
+
+        Running kmeans on a large dataset can be slow.  To speed this up we
+        run kmeans on a random sample of the data.  This parameter controls the
+        size of the sample.  The total number of vectors used to train the index
+        is `sample_rate * num_partitions`.
+
+        Increasing this value might improve the quality of the index but in
+        most cases the default should be sufficient.
+
+    m, default 20
+
+        The number of neighbors to select for each vector in the HNSW graph.
+
+        This value controls the tradeoff between search speed and accuracy.
+        The higher the value the more accurate the search but the slower it will be.
+
+    ef_construction, default 300
+
+        The number of candidates to evaluate during the construction of the HNSW graph.
+
+        This value controls the tradeoff between build speed and accuracy.
+        The higher the value the more accurate the build but the slower it will be.
+        150 to 300 is the typical range. 100 is a minimum for good quality search
+        results. In most cases, there is no benefit to setting this higher than 500.
+        This value should be set to a value that is not less than `ef` in the
+        search phase.
+    """
 
     def __init__(
         self,
@@ -108,7 +209,93 @@ class HnswPq:
 
 
 class HnswSq:
-    """Describe a HNSW-SQ index configuration."""
+    """Describe a HNSW-SQ index configuration.
+
+    HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+    It is a variant of the HNSW algorithm that uses scalar quantization to compress
+    the vectors.
+
+    Parameters
+    ----------
+
+    distance_type: str, default "L2"
+
+        The distance metric used to train the index.
+
+        The following distance types are available:
+
+        "l2" - Euclidean distance. This is a very common distance metric that
+        accounts for both magnitude and direction when determining the distance
+        between vectors. L2 distance has a range of [0, ∞).
+
+        "cosine" - Cosine distance.  Cosine distance is a distance metric
+        calculated from the cosine similarity between two vectors. Cosine
+        similarity is a measure of similarity between two non-zero vectors of an
+        inner product space. It is defined to equal the cosine of the angle
+        between them.  Unlike L2, the cosine distance is not affected by the
+        magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+        "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+        distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+        L2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+    num_partitions, default sqrt(num_rows)
+
+        The number of IVF partitions to create.
+
+        For HNSW, we recommend a small number of partitions. Setting this to 1 works
+        well for most tables. For very large tables, training just one HNSW graph
+        will require too much memory. Each partition becomes its own HNSW graph, so
+        setting this value higher reduces the peak memory use of training.
+
+    max_iterations, default 50
+
+        Max iterations to train kmeans.
+
+        When training an IVF index we use kmeans to calculate the partitions.
+        This parameter controls how many iterations of kmeans to run.
+
+        Increasing this might improve the quality of the index but in most cases
+        the parameter is unused because kmeans will converge with fewer iterations.
+        The parameter is only used in cases where kmeans does not appear to converge.
+        In those cases it is unlikely that setting this larger will lead to
+        the index converging anyways.
+
+    sample_rate, default 256
+
+        The rate used to calculate the number of training vectors for kmeans.
+
+        When an IVF index is trained, we need to calculate partitions.  These
+        are groups of vectors that are similar to each other.  To do this
+        we use an algorithm called kmeans.
+
+        Running kmeans on a large dataset can be slow.  To speed this up we
+        run kmeans on a random sample of the data.  This parameter controls the
+        size of the sample.  The total number of vectors used to train the index
+        is `sample_rate * num_partitions`.
+
+        Increasing this value might improve the quality of the index but in
+        most cases the default should be sufficient.
+
+    m, default 20
+
+        The number of neighbors to select for each vector in the HNSW graph.
+
+        This value controls the tradeoff between search speed and accuracy.
+        The higher the value the more accurate the search but the slower it will be.
+
+    ef_construction, default 300
+
+        The number of candidates to evaluate during the construction of the HNSW graph.
+
+        This value controls the tradeoff between build speed and accuracy.
+        The higher the value the more accurate the build but the slower it will be.
+        150 to 300 is the typical range. 100 is a minimum for good quality search
+        results. In most cases, there is no benefit to setting this higher than 500.
+        This value should be set to a value that is not less than `ef` in the search
+        phase.
+
+    """
 
     def __init__(
         self,

python/python/lancedb/merge.py

@@ -104,4 +104,4 @@ class LanceMergeInsertBuilder(object):
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         """
-        self._table._do_merge(self, new_data, on_bad_vectors, fill_value)
+        return self._table._do_merge(self, new_data, on_bad_vectors, fill_value)

python/python/lancedb/query.py

@@ -36,6 +36,7 @@ from . import __version__
 from .arrow import AsyncRecordBatchReader
 from .rerankers.base import Reranker
 from .rerankers.rrf import RRFReranker
+from .rerankers.util import check_reranker_result
 from .util import safe_import_pandas
 
 if TYPE_CHECKING:
@@ -575,12 +576,12 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         self._reranker = None
         self._str_query = str_query
 
-    def metric(self, metric: Literal["L2", "cosine"]) -> LanceVectorQueryBuilder:
+    def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceVectorQueryBuilder:
         """Set the distance metric to use.
 
         Parameters
         ----------
-        metric: "L2" or "cosine"
+        metric: "L2" or "cosine" or "dot"
             The distance metric to use. By default "L2" is used.
 
         Returns
@@ -588,7 +589,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
-        self._metric = metric
+        self._metric = metric.lower()
         return self
 
     def nprobes(self, nprobes: int) -> LanceVectorQueryBuilder:
@@ -679,6 +680,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         if self._reranker is not None:
             rs_table = result_set.read_all()
             result_set = self._reranker.rerank_vector(self._str_query, rs_table)
+            check_reranker_result(result_set)
             # convert result_set back to RecordBatchReader
             result_set = pa.RecordBatchReader.from_batches(
                 result_set.schema, result_set.to_batches()
@@ -811,6 +813,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
         results = results.read_all()
         if self._reranker is not None:
             results = self._reranker.rerank_fts(self._query, results)
+            check_reranker_result(results)
         return results
 
     def tantivy_to_arrow(self) -> pa.Table:
@@ -953,8 +956,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
     def __init__(
         self,
         table: "Table",
-        query: str = None,
+        query: Optional[str] = None,
-        vector_column: str = None,
+        vector_column: Optional[str] = None,
         fts_columns: Union[str, List[str]] = [],
     ):
         super().__init__(table)
@@ -1060,10 +1063,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             self._fts_query._query, vector_results, fts_results
         )
 
-        if not isinstance(results, pa.Table):  # Enforce type
+        check_reranker_result(results)
-            raise TypeError(
-                f"rerank_hybrid must return a pyarrow.Table, got {type(results)}"
-            )
 
         # apply limit after reranking
         results = results.slice(length=self._limit)
@@ -1112,8 +1112,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
 
     def rerank(
         self,
-        normalize="score",
         reranker: Reranker = RRFReranker(),
+        normalize: str = "score",
     ) -> LanceHybridQueryBuilder:
         """
         Rerank the hybrid search results using the specified reranker. The reranker
@@ -1121,12 +1121,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
 
         Parameters
         ----------
+        reranker: Reranker, default RRFReranker()
+            The reranker to use. Must be an instance of Reranker class.
         normalize: str, default "score"
             The method to normalize the scores. Can be "rank" or "score". If "rank",
             the scores are converted to ranks and then normalized. If "score", the
             scores are normalized directly.
-        reranker: Reranker, default RRFReranker()
-            The reranker to use. Must be an instance of Reranker class.
         Returns
         -------
         LanceHybridQueryBuilder

python/python/lancedb/remote/__init__.py

@@ -12,9 +12,12 @@
 #  limitations under the License.
 
 import abc
+from dataclasses import dataclass
+from datetime import timedelta
 from typing import List, Optional
 
 import attrs
+from lancedb import __version__
 import pyarrow as pa
 from pydantic import BaseModel
 
@@ -62,3 +65,109 @@ class LanceDBClient(abc.ABC):
     def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
         """Query the LanceDB server for the given table and query."""
         pass
+
+
+@dataclass
+class TimeoutConfig:
+    """Timeout configuration for remote HTTP client.
+
+    Attributes
+    ----------
+    connect_timeout: Optional[timedelta]
+        The timeout for establishing a connection. Default is 120 seconds (2 minutes).
+        This can also be set via the environment variable
+        `LANCE_CLIENT_CONNECT_TIMEOUT`, as an integer number of seconds.
+    read_timeout: Optional[timedelta]
+        The timeout for reading data from the server. Default is 300 seconds
+        (5 minutes). This can also be set via the environment variable
+        `LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
+    pool_idle_timeout: Optional[timedelta]
+        The timeout for keeping idle connections in the connection pool. Default
+        is 300 seconds (5 minutes). This can also be set via the environment variable
+        `LANCE_CLIENT_CONNECTION_TIMEOUT`, as an integer number of seconds.
+    """
+
+    connect_timeout: Optional[timedelta] = None
+    read_timeout: Optional[timedelta] = None
+    pool_idle_timeout: Optional[timedelta] = None
+
+    @staticmethod
+    def __to_timedelta(value) -> Optional[timedelta]:
+        if value is None:
+            return None
+        elif isinstance(value, timedelta):
+            return value
+        elif isinstance(value, (int, float)):
+            return timedelta(seconds=value)
+        else:
+            raise ValueError(
+                f"Invalid value for timeout: {value}, must be a timedelta "
+                "or number of seconds"
+            )
+
+    def __post_init__(self):
+        self.connect_timeout = self.__to_timedelta(self.connect_timeout)
+        self.read_timeout = self.__to_timedelta(self.read_timeout)
+        self.pool_idle_timeout = self.__to_timedelta(self.pool_idle_timeout)
+
+
+@dataclass
+class RetryConfig:
+    """Retry configuration for the remote HTTP client.
+
+    Attributes
+    ----------
+    retries: Optional[int]
+        The maximum number of retries for a request. Default is 3. You can also set this
+        via the environment variable `LANCE_CLIENT_MAX_RETRIES`.
+    connect_retries: Optional[int]
+        The maximum number of retries for connection errors. Default is 3. You can also
+        set this via the environment variable `LANCE_CLIENT_CONNECT_RETRIES`.
+    read_retries: Optional[int]
+        The maximum number of retries for read errors. Default is 3. You can also set
+        this via the environment variable `LANCE_CLIENT_READ_RETRIES`.
+    backoff_factor: Optional[float]
+        The backoff factor to apply between retries. Default is 0.25. Between each retry
+        the client will wait for the amount of seconds:
+        `{backoff factor} * (2 ** ({number of previous retries}))`. So for the default
+        of 0.25, the first retry will wait 0.25 seconds, the second retry will wait 0.5
+        seconds, the third retry will wait 1 second, etc.
+
+        You can also set this via the environment variable
+        `LANCE_CLIENT_RETRY_BACKOFF_FACTOR`.
+    backoff_jitter: Optional[float]
+        The jitter to apply to the backoff factor, in seconds. Default is 0.25.
+
+        A random value between 0 and `backoff_jitter` will be added to the backoff
+        factor in seconds. So for the default of 0.25 seconds, between 0 and 250
+        milliseconds will be added to the sleep between each retry.
+
+        You can also set this via the environment variable
+        `LANCE_CLIENT_RETRY_BACKOFF_JITTER`.
+    statuses: Optional[List[int]
+        The HTTP status codes for which to retry the request. Default is
+        [429, 500, 502, 503].
+
+        You can also set this via the environment variable
+        `LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.
+    """
+
+    retries: Optional[int] = None
+    connect_retries: Optional[int] = None
+    read_retries: Optional[int] = None
+    backoff_factor: Optional[float] = None
+    backoff_jitter: Optional[float] = None
+    statuses: Optional[List[int]] = None
+
+
+@dataclass
+class ClientConfig:
+    user_agent: str = f"LanceDB-Python-Client/{__version__}"
+    retry_config: Optional[RetryConfig] = None
+    timeout_config: Optional[TimeoutConfig] = None
+
+    def __post_init__(self):
+        if isinstance(self.retry_config, dict):
+            self.retry_config = RetryConfig(**self.retry_config)
+        if isinstance(self.timeout_config, dict):
+            self.timeout_config = TimeoutConfig(**self.timeout_config)

python/python/lancedb/remote/client.py

@@ -79,6 +79,13 @@ class RestfulLanceDBClient:
             or f"https://{self.db_name}.{self.region}.api.lancedb.com"
         )
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False  # Do not suppress exceptions
+
     def close(self):
         self.session.close()
         self.closed = True
@@ -96,19 +103,29 @@ class RestfulLanceDBClient:
 
     @staticmethod
     def _check_status(resp: requests.Response):
+        # Leaving request id empty for now, as we'll be replacing this impl
+        # with the Rust one shortly.
         if resp.status_code == 404:
-            raise LanceDBClientError(f"Not found: {resp.text}")
+            raise LanceDBClientError(
+                f"Not found: {resp.text}", request_id="", status_code=404
+            )
         elif 400 <= resp.status_code < 500:
             raise LanceDBClientError(
-                f"Bad Request: {resp.status_code}, error: {resp.text}"
+                f"Bad Request: {resp.status_code}, error: {resp.text}",
+                request_id="",
+                status_code=resp.status_code,
             )
         elif 500 <= resp.status_code < 600:
             raise LanceDBClientError(
-                f"Internal Server Error: {resp.status_code}, error: {resp.text}"
+                f"Internal Server Error: {resp.status_code}, error: {resp.text}",
+                request_id="",
+                status_code=resp.status_code,
             )
         elif resp.status_code != 200:
             raise LanceDBClientError(
-                f"Unknown Error: {resp.status_code}, error: {resp.text}"
+                f"Unknown Error: {resp.status_code}, error: {resp.text}",
+                request_id="",
+                status_code=resp.status_code,
             )
 
     @_check_not_closed

python/python/lancedb/remote/errors.py

@@ -12,5 +12,102 @@
 #  limitations under the License.
 
 
+from typing import Optional
+
+
 class LanceDBClientError(RuntimeError):
+    """An error that occurred in the LanceDB client.
+
+    Attributes
+    ----------
+    message: str
+        The error message.
+    request_id: str
+        The id of the request that failed. This can be provided in error reports
+        to help diagnose the issue.
+    status_code: int
+        The HTTP status code of the response. May be None if the request
+        failed before the response was received.
+    """
+
+    def __init__(
+        self, message: str, request_id: str, status_code: Optional[int] = None
+    ):
+        super().__init__(message)
+        self.request_id = request_id
+        self.status_code = status_code
+
+
+class HttpError(LanceDBClientError):
+    """An error that occurred during an HTTP request.
+
+    Attributes
+    ----------
+    message: str
+        The error message.
+    request_id: str
+        The id of the request that failed. This can be provided in error reports
+        to help diagnose the issue.
+    status_code: int
+        The HTTP status code of the response. May be None if the request
+        failed before the response was received.
+    """
+
     pass
+
+
+class RetryError(LanceDBClientError):
+    """An error that occurs when the client has exceeded the maximum number of retries.
+
+    The retry strategy can be adjusted by setting the
+    [retry_config](lancedb.remote.ClientConfig.retry_config) in the client
+    configuration. This is passed in the `client_config` argument of
+    [connect](lancedb.connect) and [connect_async](lancedb.connect_async).
+
+    The __cause__ attribute of this exception will be the last exception that
+    caused the retry to fail. It will be an
+    [HttpError][lancedb.remote.errors.HttpError] instance.
+
+    Attributes
+    ----------
+    message: str
+        The retry error message, which will describe which retry limit was hit.
+    request_id: str
+        The id of the request that failed. This can be provided in error reports
+        to help diagnose the issue.
+    request_failures: int
+        The number of request failures.
+    connect_failures: int
+        The number of connect failures.
+    read_failures: int
+        The number of read failures.
+    max_request_failures: int
+        The maximum number of request failures.
+    max_connect_failures: int
+        The maximum number of connect failures.
+    max_read_failures: int
+        The maximum number of read failures.
+    status_code: int
+        The HTTP status code of the last response. May be None if the request
+        failed before the response was received.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        request_id: str,
+        request_failures: int,
+        connect_failures: int,
+        read_failures: int,
+        max_request_failures: int,
+        max_connect_failures: int,
+        max_read_failures: int,
+        status_code: Optional[int],
+    ):
+        super().__init__(message, request_id, status_code)
+        self.request_failures = request_failures
+        self.connect_failures = connect_failures
+        self.read_failures = read_failures
+        self.max_request_failures = max_request_failures
+        self.max_connect_failures = max_connect_failures
+        self.max_read_failures = max_read_failures

python/python/lancedb/remote/table.py

@@ -26,7 +26,7 @@ from lancedb.embeddings import EmbeddingFunctionRegistry
 
 from ..query import LanceVectorQueryBuilder, LanceQueryBuilder
 from ..table import Query, Table, _sanitize_data
-from ..util import inf_vector_column_query, value_to_sql
+from ..util import value_to_sql, infer_vector_column_name
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -266,7 +266,7 @@ class RemoteTable(Table):
 
     def search(
         self,
-        query: Union[VEC, str],
+        query: Union[VEC, str] = None,
         vector_column_name: Optional[str] = None,
         query_type="auto",
         fts_columns: Optional[Union[str, List[str]]] = None,
@@ -305,8 +305,6 @@ class RemoteTable(Table):
             - *default None*.
             Acceptable types are: list, np.ndarray, PIL.Image.Image
 
-            - If None then the select/where/limit clauses are applied to filter
-            the table
         vector_column_name: str, optional
             The name of the vector column to search.
 
@@ -329,11 +327,15 @@ class RemoteTable(Table):
             - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
-        if vector_column_name is None and query is not None and query_type != "fts":
+        # empty query builder is not supported in saas, raise error
-            try:
+        if query is None and query_type != "hybrid":
-                vector_column_name = inf_vector_column_query(self.schema)
+            raise ValueError("Empty query is not supported")
-            except Exception as e:
+        vector_column_name = infer_vector_column_name(
-                raise e
+            schema=self.schema,
+            query_type=query_type,
+            query=query,
+            vector_column_name=vector_column_name,
+        )
 
         return LanceQueryBuilder.create(
             self,

python/python/lancedb/rerankers/answerdotai.py

@@ -32,6 +32,9 @@ class AnswerdotaiRerankers(Reranker):
         The name of the column to use as input to the cross encoder model.
     return_score : str, default "relevance"
         options are "relevance" or "all". Only "relevance" is supported for now.
+    **kwargs
+        Additional keyword arguments to pass to the model. For example, 'device'.
+        See AnswerDotAI/rerankers for more information.
     """
 
     def __init__(
@@ -40,13 +43,14 @@ class AnswerdotaiRerankers(Reranker):
         model_name: str = "answerdotai/answerai-colbert-small-v1",
         column: str = "text",
         return_score="relevance",
+        **kwargs,
     ):
         super().__init__(return_score)
         self.column = column
         rerankers = attempt_import_or_raise(
             "rerankers"
         )  # import here for faster ops later
-        self.reranker = rerankers.Reranker(model_name, model_type)
+        self.reranker = rerankers.Reranker(model_name, model_type, **kwargs)
 
     def _rerank(self, result_set: pa.Table, query: str):
         docs = result_set[self.column].to_pylist()

python/python/lancedb/rerankers/base.py

@@ -105,7 +105,7 @@ class Reranker(ABC):
         query: str,
         vector_results: pa.Table,
         fts_results: pa.Table,
-    ):
+    ) -> pa.Table:
         """
         Rerank function receives the individual results from the vector and FTS search
         results. You can choose to use any of the results to generate the final results,

python/python/lancedb/rerankers/colbert.py

@@ -26,6 +26,9 @@ class ColbertReranker(AnswerdotaiRerankers):
         The name of the column to use as input to the cross encoder model.
     return_score : str, default "relevance"
         options are "relevance" or "all". Only "relevance" is supported for now.
+    **kwargs
+        Additional keyword arguments to pass to the model, for example, 'device'.
+        See AnswerDotAI/rerankers for more information.
     """
 
     def __init__(
@@ -33,10 +36,12 @@ class ColbertReranker(AnswerdotaiRerankers):
         model_name: str = "colbert-ir/colbertv2.0",
         column: str = "text",
         return_score="relevance",
+        **kwargs,
     ):
         super().__init__(
             model_type="colbert",
             model_name=model_name,
             column=column,
             return_score=return_score,
+            **kwargs,
         )

python/python/lancedb/rerankers/linear_combination.py

@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from numpy import NaN
 import pyarrow as pa
 
 from .base import Reranker
@@ -58,14 +59,42 @@ class LinearCombinationReranker(Reranker):
     def merge_results(
         self, vector_results: pa.Table, fts_results: pa.Table, fill: float
     ):
-        # If both are empty then just return an empty table
+        # If one is empty then return the other and add _relevance_score
-        if len(vector_results) == 0 and len(fts_results) == 0:
+        # column equal the existing vector or fts score
-            return vector_results
-        # If one is empty then return the other
         if len(vector_results) == 0:
-            return fts_results
+            results = fts_results.append_column(
+                "_relevance_score",
+                pa.array(fts_results["_score"], type=pa.float32()),
+            )
+            if self.score == "relevance":
+                results = self._keep_relevance_score(results)
+            elif self.score == "all":
+                results = results.append_column(
+                    "_distance",
+                    pa.array([NaN] * len(fts_results), type=pa.float32()),
+                )
+            return results
+
         if len(fts_results) == 0:
-            return vector_results
+            # invert the distance to relevance score
+            results = vector_results.append_column(
+                "_relevance_score",
+                pa.array(
+                    [
+                        self._invert_score(distance)
+                        for distance in vector_results["_distance"].to_pylist()
+                    ],
+                    type=pa.float32(),
+                ),
+            )
+            if self.score == "relevance":
+                results = self._keep_relevance_score(results)
+            elif self.score == "all":
+                results = results.append_column(
+                    "_score",
+                    pa.array([NaN] * len(vector_results), type=pa.float32()),
+                )
+            return results
 
         # sort both input tables on _rowid
         combined_list = []

python/python/lancedb/rerankers/util.py

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+import pyarrow as pa
+
+
+def check_reranker_result(result):
+    if not isinstance(result, pa.Table):  # Enforce type
+        raise TypeError(
+            f"rerank_hybrid must return a pyarrow.Table, got {type(result)}"
+        )
+
+    # Enforce that `_relevance_score` column is present in the result of every
+    # rerank_hybrid method
+    if "_relevance_score" not in result.column_names:
+        raise ValueError(
+            "rerank_hybrid must return a pyarrow.Table with a column"
+            "named `_relevance_score`"
+        )

python/python/lancedb/table.py

@@ -19,27 +19,37 @@ from typing import (
     Optional,
     Tuple,
     Union,
+    overload,
 )
 from urllib.parse import urlparse
 
 import lance
+from .dependencies import _check_for_pandas
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.fs as pa_fs
 from lance import LanceDataset
 from lance.dependencies import _check_for_hugging_face
-from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
 from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
 from .merge import LanceMergeInsertBuilder
 from .pydantic import LanceModel, model_to_dict
-from .query import AsyncQuery, AsyncVectorQuery, LanceQueryBuilder, Query
+from .query import (
+    AsyncQuery,
+    AsyncVectorQuery,
+    LanceEmptyQueryBuilder,
+    LanceFtsQueryBuilder,
+    LanceHybridQueryBuilder,
+    LanceQueryBuilder,
+    LanceVectorQueryBuilder,
+    Query,
+)
 from .util import (
     fs_from_uri,
     get_uri_scheme,
-    inf_vector_column_query,
+    infer_vector_column_name,
     join_uri,
     safe_import_pandas,
     safe_import_polars,
@@ -53,82 +63,98 @@ if TYPE_CHECKING:
     from .db import LanceDBConnection
     from .index import BTree, IndexConfig, IvfPq, Bitmap, LabelList, FTS
 
-
 pd = safe_import_pandas()
 pl = safe_import_polars()
 
+QueryType = Literal["vector", "fts", "hybrid", "auto"]
 
-def _sanitize_data(
+
-    data,
+def _coerce_to_table(data, schema: Optional[pa.Schema] = None) -> pa.Table:
-    schema: Optional[pa.Schema],
-    metadata: Optional[dict],
-    on_bad_vectors: str,
-    fill_value: Any,
-):
     if _check_for_hugging_face(data):
         # Huggingface datasets
         from lance.dependencies import datasets
 
-        if isinstance(data, datasets.dataset_dict.DatasetDict):
+        if isinstance(data, datasets.Dataset):
-            if schema is None:
-                schema = _schema_from_hf(data, schema)
-            data = _to_record_batch_generator(
-                _to_batches_with_split(data),
-                schema,
-                metadata,
-                on_bad_vectors,
-                fill_value,
-            )
-        elif isinstance(data, datasets.Dataset):
             if schema is None:
                 schema = data.features.arrow_schema
-            data = _to_record_batch_generator(
+            return pa.Table.from_batches(data.data.to_batches(), schema=schema)
-                data.data.to_batches(), schema, metadata, on_bad_vectors, fill_value
+        elif isinstance(data, datasets.dataset_dict.DatasetDict):
-            )
+            if schema is None:
+                schema = _schema_from_hf(data, schema)
+            return pa.Table.from_batches(_to_batches_with_split(data), schema=schema)
 
     if isinstance(data, LanceModel):
         raise ValueError("Cannot add a single LanceModel to a table. Use a list.")
 
+    if isinstance(data, dict):
+        raise ValueError("Cannot add a single dictionary to a table. Use a list.")
+
     if isinstance(data, list):
         # convert to list of dict if data is a bunch of LanceModels
         if isinstance(data[0], LanceModel):
             if schema is None:
                 schema = data[0].__class__.to_arrow_schema()
             data = [model_to_dict(d) for d in data]
-            data = pa.Table.from_pylist(data, schema=schema)
+            return pa.Table.from_pylist(data, schema=schema)
+        elif isinstance(data[0], pa.RecordBatch):
+            return pa.Table.from_batches(data, schema=schema)
         else:
-            data = pa.Table.from_pylist(data)
+            return pa.Table.from_pylist(data)
-    elif isinstance(data, dict):
+    elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
-        data = vec_to_table(data)
+        # Do not add schema here, since schema may contains the vector column
-    elif pd is not None and isinstance(data, pd.DataFrame):
+        table = pa.Table.from_pandas(data, preserve_index=False)
-        data = pa.Table.from_pandas(data, preserve_index=False)
         # Do not serialize Pandas metadata
-        meta = data.schema.metadata if data.schema.metadata is not None else {}
+        meta = table.schema.metadata if table.schema.metadata is not None else {}
         meta = {k: v for k, v in meta.items() if k != b"pandas"}
-        data = data.replace_schema_metadata(meta)
+        return table.replace_schema_metadata(meta)
-    elif pl is not None and isinstance(data, pl.DataFrame):
+    elif isinstance(data, pa.Table):
-        data = data.to_arrow()
+        return data
-
+    elif isinstance(data, pa.RecordBatch):
-    if isinstance(data, pa.Table):
+        return pa.Table.from_batches([data])
-        if metadata:
+    elif isinstance(data, LanceDataset):
-            data = _append_vector_col(data, metadata, schema)
+        return data.scanner().to_table()
-            metadata.update(data.schema.metadata or {})
+    elif isinstance(data, pa.dataset.Dataset):
-            data = data.replace_schema_metadata(metadata)
+        return data.to_table()
-        data = _sanitize_schema(
+    elif isinstance(data, pa.dataset.Scanner):
-            data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        return data.to_table()
-        )
+    elif isinstance(data, pa.RecordBatchReader):
-        if schema is None:
+        return data.read_all()
-            schema = data.schema
+    elif (
+        type(data).__module__.startswith("polars")
+        and data.__class__.__name__ == "DataFrame"
+    ):
+        return data.to_arrow()
     elif isinstance(data, Iterable):
-        data = _to_record_batch_generator(
+        return _process_iterator(data, schema)
-            data, schema, metadata, on_bad_vectors, fill_value
-        )
-        if schema is None:
-            data, schema = _generator_to_data_and_schema(data)
-            if schema is None:
-                raise ValueError("Cannot infer schema from generator data")
     else:
-        raise TypeError(f"Unsupported data type: {type(data)}")
+        raise TypeError(
+            f"Unknown data type {type(data)}. "
+            "Please check "
+            "https://lancedb.github.io/lancedb/python/python/ "
+            "to see supported types."
+        )
+
+
+def _sanitize_data(
+    data: Any,
+    schema: Optional[pa.Schema] = None,
+    metadata: Optional[dict] = None,  # embedding metadata
+    on_bad_vectors: str = "error",
+    fill_value: float = 0.0,
+):
+    data = _coerce_to_table(data, schema)
+
+    if metadata:
+        data = _append_vector_col(data, metadata, schema)
+        metadata.update(data.schema.metadata or {})
+        data = data.replace_schema_metadata(metadata)
+
+    # TODO improve the logics in _sanitize_schema
+    data = _sanitize_schema(data, schema, on_bad_vectors, fill_value)
+    if schema is None:
+        schema = data.schema
+
+    _validate_schema(schema)
     return data, schema
 
 
@@ -149,6 +175,9 @@ def sanitize_create_table(
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
         )
+    else:
+        if schema is not None:
+            data = pa.Table.from_pylist([], schema)
     if schema is None:
         if data is None:
             raise ValueError("Either data or schema must be provided")
@@ -505,7 +534,7 @@ class Table(ABC):
             Only available with use_tantivy=False
             If False, do not store the positions of the terms in the text.
             This can reduce the size of the index and improve indexing speed.
-            But it will not be possible to use phrase queries.
+            But it will raise an exception for phrase queries.
 
         """
         raise NotImplementedError
@@ -525,7 +554,7 @@ class Table(ABC):
         data: DATA
             The data to insert into the table. Acceptable types are:
 
-            - dict or list-of-dict
+            - list-of-dict
 
             - pandas.DataFrame
 
@@ -607,7 +636,7 @@ class Table(ABC):
         self,
         query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
         vector_column_name: Optional[str] = None,
-        query_type: str = "auto",
+        query_type: QueryType = "auto",
         ordering_field_name: Optional[str] = None,
         fts_columns: Optional[Union[str, List[str]]] = None,
     ) -> LanceQueryBuilder:
@@ -1380,7 +1409,7 @@ class LanceTable(Table):
 
         Parameters
         ----------
-        data: list-of-dict, dict, pd.DataFrame
+        data: list-of-dict, pd.DataFrame
             The data to insert into the table.
         mode: str
             The mode to use when writing the data. Valid values are
@@ -1487,11 +1516,51 @@ class LanceTable(Table):
             self.schema.metadata
         )
 
+    @overload
     def search(
         self,
         query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
         vector_column_name: Optional[str] = None,
-        query_type: str = "auto",
+        query_type: Literal["vector"] = "vector",
+        ordering_field_name: Optional[str] = None,
+        fts_columns: Optional[Union[str, List[str]]] = None,
+    ) -> LanceVectorQueryBuilder: ...
+
+    @overload
+    def search(
+        self,
+        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        vector_column_name: Optional[str] = None,
+        query_type: Literal["fts"] = "fts",
+        ordering_field_name: Optional[str] = None,
+        fts_columns: Optional[Union[str, List[str]]] = None,
+    ) -> LanceFtsQueryBuilder: ...
+
+    @overload
+    def search(
+        self,
+        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        vector_column_name: Optional[str] = None,
+        query_type: Literal["hybrid"] = "hybrid",
+        ordering_field_name: Optional[str] = None,
+        fts_columns: Optional[Union[str, List[str]]] = None,
+    ) -> LanceHybridQueryBuilder: ...
+
+    @overload
+    def search(
+        self,
+        query: None = None,
+        vector_column_name: Optional[str] = None,
+        query_type: QueryType = "auto",
+        ordering_field_name: Optional[str] = None,
+        fts_columns: Optional[Union[str, List[str]]] = None,
+    ) -> LanceEmptyQueryBuilder: ...
+
+    def search(
+        self,
+        query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
+        vector_column_name: Optional[str] = None,
+        query_type: QueryType = "auto",
         ordering_field_name: Optional[str] = None,
         fts_columns: Optional[Union[str, List[str]]] = None,
     ) -> LanceQueryBuilder:
@@ -1561,11 +1630,12 @@ class LanceTable(Table):
             and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
-        if vector_column_name is None and query is not None and query_type != "fts":
+        vector_column_name = infer_vector_column_name(
-            try:
+            schema=self.schema,
-                vector_column_name = inf_vector_column_query(self.schema)
+            query_type=query_type,
-            except Exception as e:
+            query=query,
-                raise e
+            vector_column_name=vector_column_name,
+        )
 
         return LanceQueryBuilder.create(
             self,
@@ -1929,22 +1999,26 @@ def _sanitize_vector_column(
                 data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             )
             vec_arr = data[vector_column_name].combine_chunks()
+        vec_arr = ensure_fixed_size_list(vec_arr)
+        data = data.set_column(
+            data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        )
     elif not pa.types.is_fixed_size_list(vec_arr.type):
         raise TypeError(f"Unsupported vector column type: {vec_arr.type}")
 
-    vec_arr = ensure_fixed_size_list(vec_arr)
+    if pa.types.is_float16(vec_arr.values.type):
-    data = data.set_column(
+        # Use numpy to check for NaNs, because as pyarrow does not have `is_nan`
-        data.column_names.index(vector_column_name), vector_column_name, vec_arr
+        # kernel over f16 types yet.
-    )
+        values_np = vec_arr.values.to_numpy(zero_copy_only=True)
-
+        if np.isnan(values_np).any():
-    # Use numpy to check for NaNs, because as pyarrow 14.0.2 does not have `is_nan`
+            data = _sanitize_nans(
-    # kernel over f16 types.
+                data, fill_value, on_bad_vectors, vec_arr, vector_column_name
-    values_np = vec_arr.values.to_numpy(zero_copy_only=False)
+            )
-    if np.isnan(values_np).any():
+    else:
-        data = _sanitize_nans(
+        if pc.any(pc.is_null(vec_arr.values, nan_is_null=True)).as_py():
-            data, fill_value, on_bad_vectors, vec_arr, vector_column_name
+            data = _sanitize_nans(
-        )
+                data, fill_value, on_bad_vectors, vec_arr, vector_column_name
-
+            )
     return data
 
 
@@ -1988,8 +2062,15 @@ def _sanitize_jagged(data, fill_value, on_bad_vectors, vec_arr, vector_column_na
     return data
 
 
-def _sanitize_nans(data, fill_value, on_bad_vectors, vec_arr, vector_column_name):
+def _sanitize_nans(
+    data,
+    fill_value,
+    on_bad_vectors,
+    vec_arr: pa.FixedSizeListArray,
+    vector_column_name: str,
+):
     """Sanitize NaNs in vectors"""
+    assert pa.types.is_fixed_size_list(vec_arr.type)
     if on_bad_vectors == "error":
         raise ValueError(
             f"Vector column {vector_column_name} has NaNs. "
@@ -2009,12 +2090,63 @@ def _sanitize_nans(data, fill_value, on_bad_vectors, vec_arr, vector_column_name
             data.column_names.index(vector_column_name), vector_column_name, vec_arr
         )
     elif on_bad_vectors == "drop":
-        is_value_nan = pc.is_nan(vec_arr.values).to_numpy(zero_copy_only=False)
+        # Drop is very slow to be able to filter out NaNs in a fixed size list array
-        is_full = np.any(~is_value_nan.reshape(-1, vec_arr.type.list_size), axis=1)
+        np_arr = np.isnan(vec_arr.values.to_numpy(zero_copy_only=False))
-        data = data.filter(is_full)
+        np_arr = np_arr.reshape(-1, vec_arr.type.list_size)
+        not_nulls = np.any(np_arr, axis=1)
+        data = data.filter(~not_nulls)
     return data
 
 
+def _validate_schema(schema: pa.Schema):
+    """
+    Make sure the metadata is valid utf8
+    """
+    if schema.metadata is not None:
+        _validate_metadata(schema.metadata)
+
+
+def _validate_metadata(metadata: dict):
+    """
+    Make sure the metadata values are valid utf8 (can be nested)
+
+    Raises ValueError if not valid utf8
+    """
+    for k, v in metadata.items():
+        if isinstance(v, bytes):
+            try:
+                v.decode("utf8")
+            except UnicodeDecodeError:
+                raise ValueError(
+                    f"Metadata key {k} is not valid utf8. "
+                    "Consider base64 encode for generic binary metadata."
+                )
+        elif isinstance(v, dict):
+            _validate_metadata(v)
+
+
+def _process_iterator(data: Iterable, schema: Optional[pa.Schema] = None) -> pa.Table:
+    batches = []
+    for batch in data:
+        batch_table = _coerce_to_table(batch, schema)
+        if schema is not None:
+            if batch_table.schema != schema:
+                try:
+                    batch_table = batch_table.cast(schema)
+                except pa.lib.ArrowInvalid:
+                    raise ValueError(
+                        f"Input iterator yielded a batch with schema that "
+                        f"does not match the expected schema.\nExpected:\n{schema}\n"
+                        f"Got:\n{batch_table.schema}"
+                    )
+        batches.append(batch_table)
+
+    if batches:
+        return pa.concat_tables(batches)
+    else:
+        raise ValueError("Input iterable is empty")
+
+
 class AsyncTable:
     """
     An AsyncTable is a collection of Records in a LanceDB Database.
@@ -2216,7 +2348,7 @@ class AsyncTable:
         data: DATA
             The data to insert into the table. Acceptable types are:
 
-            - dict or list-of-dict
+            - list-of-dict
 
             - pandas.DataFrame
 
@@ -2332,7 +2464,31 @@ class AsyncTable:
         on_bad_vectors: str,
         fill_value: float,
     ):
-        pass
+        schema = await self.schema()
+        if on_bad_vectors is None:
+            on_bad_vectors = "error"
+        if fill_value is None:
+            fill_value = 0.0
+        data, _ = _sanitize_data(
+            new_data,
+            schema,
+            metadata=schema.metadata,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+        )
+        if isinstance(data, pa.Table):
+            data = pa.RecordBatchReader.from_batches(data.schema, data.to_batches())
+        await self._inner.execute_merge_insert(
+            data,
+            dict(
+                on=merge._on,
+                when_matched_update_all=merge._when_matched_update_all,
+                when_matched_update_all_condition=merge._when_matched_update_all_condition,
+                when_not_matched_insert_all=merge._when_not_matched_insert_all,
+                when_not_matched_by_source_delete=merge._when_not_matched_by_source_delete,
+                when_not_matched_by_source_condition=merge._when_not_matched_by_source_condition,
+            ),
+        )
 
     async def delete(self, where: str):
         """Delete rows from the table.
@@ -2551,6 +2707,26 @@ class AsyncTable:
         """
         return await self._inner.list_indices()
 
+    async def index_stats(self, index_name: str) -> Optional[IndexStatistics]:
+        """
+        Retrieve statistics about an index
+
+        Parameters
+        ----------
+        index_name: str
+            The name of the index to retrieve statistics for
+
+        Returns
+        -------
+        IndexStatistics or None
+            The statistics about the index. Returns None if the index does not exist.
+        """
+        stats = await self._inner.index_stats(index_name)
+        if stats is None:
+            return None
+        else:
+            return IndexStatistics(**stats)
+
     async def uses_v2_manifest_paths(self) -> bool:
         """
         Check if the table is using the new v2 manifest paths.
@@ -2581,3 +2757,31 @@ class AsyncTable:
         to check if the table is already using the new path style.
         """
         await self._inner.migrate_manifest_paths_v2()
+
+
+@dataclass
+class IndexStatistics:
+    """
+    Statistics about an index.
+
+    Attributes
+    ----------
+    num_indexed_rows: int
+        The number of rows that are covered by this index.
+    num_unindexed_rows: int
+        The number of rows that are not covered by this index.
+    index_type: str
+        The type of index that was created.
+    distance_type: Optional[str]
+        The distance type used by the index.
+    num_indices: Optional[int]
+        The number of parts the index is split into.
+    """
+
+    num_indexed_rows: int
+    num_unindexed_rows: int
+    index_type: Literal[
+        "IVF_PQ", "IVF_HNSW_PQ", "IVF_HNSW_SQ", "FTS", "BTREE", "BITMAP", "LABEL_LIST"
+    ]
+    distance_type: Optional[Literal["l2", "cosine", "dot"]] = None
+    num_indices: Optional[int] = None

python/python/lancedb/util.py

@@ -9,7 +9,7 @@ import pathlib
 import warnings
 from datetime import date, datetime
 from functools import singledispatch
-from typing import Tuple, Union
+from typing import Tuple, Union, Optional, Any
 from urllib.parse import urlparse
 
 import numpy as np
@@ -212,6 +212,23 @@ def inf_vector_column_query(schema: pa.Schema) -> str:
     return vector_col_name
 
 
+def infer_vector_column_name(
+    schema: pa.Schema,
+    query_type: str,
+    query: Optional[Any],  # inferred later in query builder
+    vector_column_name: Optional[str],
+):
+    if (vector_column_name is None and query is not None and query_type != "fts") or (
+        vector_column_name is None and query_type == "hybrid"
+    ):
+        try:
+            vector_column_name = inf_vector_column_query(schema)
+        except Exception as e:
+            raise e
+
+    return vector_column_name
+
+
 @singledispatch
 def value_to_sql(value):
     raise NotImplementedError("SQL conversion is not implemented for this type")
@@ -219,6 +236,7 @@ def value_to_sql(value):
 
 @value_to_sql.register(str)
 def _(value: str):
+    value = value.replace("'", "''")
     return f"'{value}'"
 
 

python/python/tests/test_db.py

@@ -354,7 +354,7 @@ async def test_create_mode_async(tmp_path):
     )
     await db.create_table("test", data=data)
 
-    with pytest.raises(RuntimeError):
+    with pytest.raises(ValueError, match="already exists"):
         await db.create_table("test", data=data)
 
     new_data = pd.DataFrame(
@@ -382,7 +382,7 @@ async def test_create_exist_ok_async(tmp_path):
     )
     tbl = await db.create_table("test", data=data)
 
-    with pytest.raises(RuntimeError):
+    with pytest.raises(ValueError, match="already exists"):
         await db.create_table("test", data=data)
 
     # open the table but don't add more rows
@@ -594,7 +594,9 @@ async def test_create_in_v2_mode(tmp_path):
     db = await lancedb.connect_async(tmp_path)
 
     # Create table in v1 mode
-    tbl = await db.create_table("test", data=make_data(), schema=schema)
+    tbl = await db.create_table(
+        "test", data=make_data(), schema=schema, data_storage_version="legacy"
+    )
 
     async def is_in_v2_mode(tbl):
         batches = await tbl.query().to_batches(max_batch_length=1024 * 10)
@@ -626,7 +628,9 @@ async def test_create_in_v2_mode(tmp_path):
     assert await is_in_v2_mode(tbl)
 
     # Create empty table uses v1 mode by default
-    tbl = await db.create_table("test_empty_v2_default", data=None, schema=schema)
+    tbl = await db.create_table(
+        "test_empty_v2_default", data=None, schema=schema, data_storage_version="legacy"
+    )
     await tbl.add(make_table())
 
     assert not await is_in_v2_mode(tbl)

python/python/tests/test_embeddings.py

@@ -86,6 +86,47 @@ def test_embedding_function(tmp_path):
     assert np.allclose(actual, expected)
 
 
+def test_embedding_with_bad_results(tmp_path):
+    @register("mock-embedding")
+    class MockEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 128
+
+        def generate_embeddings(
+            self, texts: Union[List[str], np.ndarray]
+        ) -> list[Union[np.array, None]]:
+            return [
+                None if i % 2 == 0 else np.random.randn(self.ndims())
+                for i in range(len(texts))
+            ]
+
+    db = lancedb.connect(tmp_path)
+    registry = EmbeddingFunctionRegistry.get_instance()
+    model = registry.get("mock-embedding").create()
+
+    class Schema(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("test", schema=Schema, mode="overwrite")
+    table.add(
+        [{"text": "hello world"}, {"text": "bar"}],
+        on_bad_vectors="drop",
+    )
+
+    df = table.to_pandas()
+    assert len(table) == 1
+    assert df.iloc[0]["text"] == "bar"
+
+    # table = db.create_table("test2", schema=Schema, mode="overwrite")
+    # table.add(
+    #     [{"text": "hello world"}, {"text": "bar"}],
+    # )
+    # assert len(table) == 2
+    # tbl = table.to_arrow()
+    # assert tbl["vector"].null_count == 1
+
+
 @pytest.mark.slow
 def test_embedding_function_rate_limit(tmp_path):
     def _get_schema_from_model(model):
@@ -142,3 +183,45 @@ def test_add_optional_vector(tmp_path):
     expected = LanceSchema(id="id", text="text")
     tbl.add([expected])
     assert not (np.abs(tbl.to_pandas()["vector"][0]) < 1e-6).all()
+
+
+@pytest.mark.parametrize(
+    "embedding_type",
+    [
+        "openai",
+        "sentence-transformers",
+        "huggingface",
+        "ollama",
+        "cohere",
+        "instructor",
+    ],
+)
+def test_embedding_function_safe_model_dump(embedding_type):
+    registry = get_registry()
+
+    # Note: Some embedding types might require specific parameters
+    try:
+        model = registry.get(embedding_type).create()
+    except Exception as e:
+        pytest.skip(f"Skipping {embedding_type} due to error: {str(e)}")
+
+    dumped_model = model.safe_model_dump()
+
+    assert all(
+        not k.startswith("_") for k in dumped_model.keys()
+    ), f"{embedding_type}: Dumped model contains keys starting with underscore"
+
+    assert (
+        "max_retries" in dumped_model
+    ), f"{embedding_type}: Essential field 'max_retries' is missing from dumped model"
+
+    assert isinstance(
+        dumped_model, dict
+    ), f"{embedding_type}: Dumped model is not a dictionary"
+
+    for key in model.__dict__:
+        if key.startswith("_"):
+            assert key not in dumped_model, (
+                f"{embedding_type}: Private attribute '{key}' "
+                f"is present in dumped model"
+            )

python/python/tests/test_embeddings_slow.py

@@ -442,3 +442,42 @@ def test_watsonx_embedding(tmp_path):
         tbl.add(df)
         assert len(tbl.to_pandas()["vector"][0]) == model.ndims()
         assert tbl.search("hello").limit(1).to_pandas()["text"][0] == "hello world"
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    importlib.util.find_spec("ollama") is None, reason="Ollama not installed"
+)
+def test_ollama_embedding(tmp_path):
+    model = get_registry().get("ollama").create(max_retries=0)
+
+    class TextModel(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+
+    assert len(tbl.to_pandas()["vector"][0]) == model.ndims()
+
+    result = tbl.search("hello").limit(1).to_pandas()
+    assert result["text"][0] == "hello world"
+
+    # Test safe_model_dump
+    dumped_model = model.safe_model_dump()
+    assert isinstance(dumped_model, dict)
+    assert "name" in dumped_model
+    assert "max_retries" in dumped_model
+    assert dumped_model["max_retries"] == 0
+    assert all(not k.startswith("_") for k in dumped_model.keys())
+
+    # Test serialization of the dumped model
+    import json
+
+    try:
+        json.dumps(dumped_model)
+    except TypeError:
+        pytest.fail("Failed to JSON serialize the dumped model")

python/python/tests/test_fts.py

@@ -143,7 +143,7 @@ def test_create_index_with_stemming(tmp_path, table):
 @pytest.mark.parametrize("with_position", [True, False])
 def test_create_inverted_index(table, use_tantivy, with_position):
     if use_tantivy and not with_position:
-        pytest.skip("we don't support to build tantivy index without position")
+        pytest.skip("we don't support building a tantivy index without position")
     table.create_fts_index("text", use_tantivy=use_tantivy, with_position=with_position)
 
 

python/python/tests/test_index.py

@@ -63,17 +63,24 @@ async def test_create_scalar_index(some_table: AsyncTable):
 @pytest.mark.asyncio
 async def test_create_bitmap_index(some_table: AsyncTable):
     await some_table.create_index("id", config=Bitmap())
-    # TODO: Fix via https://github.com/lancedb/lance/issues/2039
+    indices = await some_table.list_indices()
-    # indices = await some_table.list_indices()
+    assert str(indices) == '[Index(Bitmap, columns=["id"])]'
-    # assert str(indices) == '[Index(Bitmap, columns=["id"])]'
+    indices = await some_table.list_indices()
+    assert len(indices) == 1
+    index_name = indices[0].name
+    stats = await some_table.index_stats(index_name)
+    assert stats.index_type == "BITMAP"
+    assert stats.distance_type is None
+    assert stats.num_indexed_rows == await some_table.count_rows()
+    assert stats.num_unindexed_rows == 0
+    assert stats.num_indices == 1
 
 
 @pytest.mark.asyncio
 async def test_create_label_list_index(some_table: AsyncTable):
     await some_table.create_index("tags", config=LabelList())
-    # TODO: Fix via https://github.com/lancedb/lance/issues/2039
+    indices = await some_table.list_indices()
-    # indices = await some_table.list_indices()
+    assert str(indices) == '[Index(LabelList, columns=["tags"])]'
-    # assert str(indices) == '[Index(LabelList, columns=["id"])]'
 
 
 @pytest.mark.asyncio
@@ -91,6 +98,14 @@ async def test_create_vector_index(some_table: AsyncTable):
     assert len(indices) == 1
     assert indices[0].index_type == "IvfPq"
     assert indices[0].columns == ["vector"]
+    assert indices[0].name == "vector_idx"
+
+    stats = await some_table.index_stats("vector_idx")
+    assert stats.index_type == "IVF_PQ"
+    assert stats.distance_type == "l2"
+    assert stats.num_indexed_rows == await some_table.count_rows()
+    assert stats.num_unindexed_rows == 0
+    assert stats.num_indices == 1
 
 
 @pytest.mark.asyncio

python/python/tests/test_remote_client.py

@@ -74,21 +74,23 @@ async def test_e2e_with_mock_server():
     await mock_server.start()
 
     try:
-        client = RestfulLanceDBClient("lancedb+http://localhost:8111")
+        with RestfulLanceDBClient("lancedb+http://localhost:8111") as client:
-        df = (
+            df = (
-            await client.query(
+                await client.query(
-                "test_table",
+                    "test_table",
-                VectorQuery(
+                    VectorQuery(
-                    vector=np.random.rand(128).tolist(),
+                        vector=np.random.rand(128).tolist(),
-                    k=10,
+                        k=10,
-                    _metric="L2",
+                        _metric="L2",
-                    columns=["id", "vector"],
+                        columns=["id", "vector"],
-                ),
+                    ),
-            )
+                )
-        ).to_pandas()
+            ).to_pandas()
 
-        assert "vector" in df.columns
+            assert "vector" in df.columns
-        assert "id" in df.columns
+            assert "id" in df.columns
+
+        assert client.closed
     finally:
         # make sure we don't leak resources
         await mock_server.stop()

python/python/tests/test_remote_db.py

@@ -1,19 +1,17 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+import contextlib
-#  You may obtain a copy of the License at
+import http.server
-#      http://www.apache.org/licenses/LICENSE-2.0
+import threading
-#
+from unittest.mock import MagicMock
-#  Unless required by applicable law or agreed to in writing, software
+import uuid
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import lancedb
+from lancedb.remote.errors import HttpError, RetryError
 import pyarrow as pa
 from lancedb.remote.client import VectorQuery, VectorQueryResult
+import pytest
 
 
 class FakeLanceDBClient:
@@ -39,3 +37,156 @@ def test_remote_db():
     table = conn["test"]
     table.schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
     table.search([1.0, 2.0]).to_pandas()
+
+
+def test_create_empty_table():
+    client = MagicMock()
+    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
+
+    conn._client = client
+
+    schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
+
+    client.post.return_value = {"status": "ok"}
+    table = conn.create_table("test", schema=schema)
+    assert table.name == "test"
+    assert client.post.call_args[0][0] == "/v1/table/test/create/"
+
+    json_schema = {
+        "fields": [
+            {
+                "name": "vector",
+                "nullable": True,
+                "type": {
+                    "type": "fixed_size_list",
+                    "fields": [
+                        {"name": "item", "nullable": True, "type": {"type": "float"}}
+                    ],
+                    "length": 2,
+                },
+            },
+        ]
+    }
+    client.post.return_value = {"schema": json_schema}
+    assert table.schema == schema
+    assert client.post.call_args[0][0] == "/v1/table/test/describe/"
+
+    client.post.return_value = 0
+    assert table.count_rows(None) == 0
+
+
+def test_create_table_with_recordbatches():
+    client = MagicMock()
+    conn = lancedb.connect("db://client-will-be-injected", api_key="fake")
+
+    conn._client = client
+
+    batch = pa.RecordBatch.from_arrays([pa.array([[1.0, 2.0], [3.0, 4.0]])], ["vector"])
+
+    client.post.return_value = {"status": "ok"}
+    table = conn.create_table("test", [batch], schema=batch.schema)
+    assert table.name == "test"
+    assert client.post.call_args[0][0] == "/v1/table/test/create/"
+
+
+def make_mock_http_handler(handler):
+    class MockLanceDBHandler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            handler(self)
+
+        def do_POST(self):
+            handler(self)
+
+    return MockLanceDBHandler
+
+
+@contextlib.asynccontextmanager
+async def mock_lancedb_connection(handler):
+    with http.server.HTTPServer(
+        ("localhost", 8080), make_mock_http_handler(handler)
+    ) as server:
+        handle = threading.Thread(target=server.serve_forever)
+        handle.start()
+
+        db = await lancedb.connect_async(
+            "db://dev",
+            api_key="fake",
+            host_override="http://localhost:8080",
+            client_config={
+                "retry_config": {"retries": 2},
+                "timeout_config": {
+                    "connect_timeout": 1,
+                },
+            },
+        )
+
+        try:
+            yield db
+        finally:
+            server.shutdown()
+            handle.join()
+
+
+@pytest.mark.asyncio
+async def test_async_remote_db():
+    def handler(request):
+        # We created a UUID request id
+        request_id = request.headers["x-request-id"]
+        assert uuid.UUID(request_id).version == 4
+
+        # We set a user agent with the current library version
+        user_agent = request.headers["User-Agent"]
+        assert user_agent == f"LanceDB-Python-Client/{lancedb.__version__}"
+
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        request.wfile.write(b'{"tables": []}')
+
+    async with mock_lancedb_connection(handler) as db:
+        table_names = await db.table_names()
+        assert table_names == []
+
+
+@pytest.mark.asyncio
+async def test_http_error():
+    request_id_holder = {"request_id": None}
+
+    def handler(request):
+        request_id_holder["request_id"] = request.headers["x-request-id"]
+
+        request.send_response(507)
+        request.end_headers()
+        request.wfile.write(b"Internal Server Error")
+
+    async with mock_lancedb_connection(handler) as db:
+        with pytest.raises(HttpError, match="Internal Server Error") as exc_info:
+            await db.table_names()
+
+        assert exc_info.value.request_id == request_id_holder["request_id"]
+        assert exc_info.value.status_code == 507
+
+
+@pytest.mark.asyncio
+async def test_retry_error():
+    request_id_holder = {"request_id": None}
+
+    def handler(request):
+        request_id_holder["request_id"] = request.headers["x-request-id"]
+
+        request.send_response(429)
+        request.end_headers()
+        request.wfile.write(b"Try again later")
+
+    async with mock_lancedb_connection(handler) as db:
+        with pytest.raises(RetryError, match="Hit retry limit") as exc_info:
+            await db.table_names()
+
+        assert exc_info.value.request_id == request_id_holder["request_id"]
+        assert exc_info.value.status_code == 429
+
+        cause = exc_info.value.__cause__
+        assert isinstance(cause, HttpError)
+        assert "Try again later" in str(cause)
+        assert cause.request_id == request_id_holder["request_id"]
+        assert cause.status_code == 429

python/python/tests/test_rerankers.py

@@ -120,12 +120,14 @@ def _run_test_reranker(reranker, table, query, query_vector, schema):
     )
 
     assert len(result) == 30
-    err = (
+    ascending_relevance_err = (
         "The _relevance_score column of the results returned by the reranker "
         "represents the relevance of the result to the query & should "
         "be descending."
     )
-    assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), err
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
 
     # Vector search setting
     result = (
@@ -135,7 +137,9 @@ def _run_test_reranker(reranker, table, query, query_vector, schema):
         .to_arrow()
     )
     assert len(result) == 30
-    assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), err
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
     result_explicit = (
         table.search(query_vector, vector_column_name="vector")
         .rerank(reranker=reranker, query_string=query)
@@ -158,7 +162,26 @@ def _run_test_reranker(reranker, table, query, query_vector, schema):
         .to_arrow()
     )
     assert len(result) > 0
-    assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), err
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
+
+    # empty FTS results
+    query = "abcxyz" * 100
+    result = (
+        table.search(query_type="hybrid", vector_column_name="vector")
+        .vector(query_vector)
+        .text(query)
+        .limit(30)
+        .rerank(reranker=reranker)
+        .to_arrow()
+    )
+
+    # should return _relevance_score column
+    assert "_relevance_score" in result.column_names
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
 
     # Multi-vector search setting
     rs1 = table.search(query, vector_column_name="vector").limit(10).with_row_id(True)
@@ -172,7 +195,7 @@ def _run_test_reranker(reranker, table, query, query_vector, schema):
     result_deduped = reranker.rerank_multivector(
         [rs1, rs2, rs1], query, deduplicate=True
     )
-    assert len(result_deduped) < 20
+    assert len(result_deduped) <= 20
     result_arrow = reranker.rerank_multivector([rs1.to_arrow(), rs2.to_arrow()], query)
     assert len(result) == 20 and result == result_arrow
 
@@ -213,7 +236,7 @@ def _run_test_hybrid_reranker(reranker, tmp_path, use_tantivy):
         .vector(query_vector)
         .text(query)
         .limit(30)
-        .rerank(normalize="score")
+        .rerank(reranker, normalize="score")
         .to_arrow()
     )
     assert len(result) == 30
@@ -228,12 +251,30 @@ def _run_test_hybrid_reranker(reranker, tmp_path, use_tantivy):
         table.search(query, query_type="hybrid", vector_column_name="vector").text(
             query
         ).to_arrow()
-
+    ascending_relevance_err = (
-    assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), (
         "The _relevance_score column of the results returned by the reranker "
         "represents the relevance of the result to the query & should "
         "be descending."
     )
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
+
+    # Test with empty FTS results
+    query = "abcxyz" * 100
+    result = (
+        table.search(query_type="hybrid", vector_column_name="vector")
+        .vector(query_vector)
+        .text(query)
+        .limit(30)
+        .rerank(reranker=reranker)
+        .to_arrow()
+    )
+    # should return _relevance_score column
+    assert "_relevance_score" in result.column_names
+    assert np.all(
+        np.diff(result.column("_relevance_score").to_numpy()) <= 0
+    ), ascending_relevance_err
 
 
 @pytest.mark.parametrize("use_tantivy", [True, False])

python/python/tests/test_table.py

@@ -64,6 +64,55 @@ def test_basic(db):
     assert table.to_lance().to_table() == ds.to_table()
 
 
+def test_input_data_type(db, tmp_path):
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("name", pa.string()),
+            pa.field("age", pa.int32()),
+        ]
+    )
+
+    data = {
+        "id": [1, 2, 3, 4, 5],
+        "name": ["Alice", "Bob", "Charlie", "David", "Eve"],
+        "age": [25, 30, 35, 40, 45],
+    }
+    record_batch = pa.RecordBatch.from_pydict(data, schema=schema)
+    pa_reader = pa.RecordBatchReader.from_batches(record_batch.schema, [record_batch])
+    pa_table = pa.Table.from_batches([record_batch])
+
+    def create_dataset(tmp_path):
+        path = os.path.join(tmp_path, "test_source_dataset")
+        pa.dataset.write_dataset(pa_table, path, format="parquet")
+        return pa.dataset.dataset(path, format="parquet")
+
+    pa_dataset = create_dataset(tmp_path)
+    pa_scanner = pa_dataset.scanner()
+
+    input_types = [
+        ("RecordBatchReader", pa_reader),
+        ("RecordBatch", record_batch),
+        ("Table", pa_table),
+        ("Dataset", pa_dataset),
+        ("Scanner", pa_scanner),
+    ]
+    for input_type, input_data in input_types:
+        table_name = f"test_{input_type.lower()}"
+        ds = LanceTable.create(db, table_name, data=input_data).to_lance()
+        assert ds.schema == schema
+        assert ds.count_rows() == 5
+
+        assert ds.schema.field("id").type == pa.int64()
+        assert ds.schema.field("name").type == pa.string()
+        assert ds.schema.field("age").type == pa.int32()
+
+        result_table = ds.to_table()
+        assert result_table.column("id").to_pylist() == data["id"]
+        assert result_table.column("name").to_pylist() == data["name"]
+        assert result_table.column("age").to_pylist() == data["age"]
+
+
 @pytest.mark.asyncio
 async def test_close(db_async: AsyncConnection):
     table = await db_async.create_table("some_table", data=[{"id": 0}])
@@ -144,6 +193,24 @@ def test_empty_table(db):
     tbl.add(data=data)
 
 
+def test_add_dictionary(db):
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+            pa.field("item", pa.string()),
+            pa.field("price", pa.float32()),
+        ]
+    )
+    tbl = LanceTable.create(db, "test", schema=schema)
+    data = {"vector": [3.1, 4.1], "item": "foo", "price": 10.0}
+    with pytest.raises(ValueError) as excep_info:
+        tbl.add(data=data)
+    assert (
+        str(excep_info.value)
+        == "Cannot add a single dictionary to a table. Use a list."
+    )
+
+
 def test_add(db):
     schema = pa.schema(
         [
@@ -274,7 +341,6 @@ def test_polars(db):
 
 
 def _add(table, schema):
-    # table = LanceTable(db, "test")
     assert len(table) == 2
 
     table.add([{"vector": [6.3, 100.5], "item": "new", "price": 30.0}])
@@ -588,11 +654,13 @@ def test_merge_insert(db):
     new_data = pa.table({"a": [2, 4], "b": ["x", "z"]})
 
     # replace-range
-    table.merge_insert(
+    (
-        "a"
+        table.merge_insert("a")
-    ).when_matched_update_all().when_not_matched_insert_all().when_not_matched_by_source_delete(
+        .when_matched_update_all()
-        "a > 2"
+        .when_not_matched_insert_all()
-    ).execute(new_data)
+        .when_not_matched_by_source_delete("a > 2")
+        .execute(new_data)
+    )
 
     expected = pa.table({"a": [1, 2, 4], "b": ["a", "x", "z"]})
     assert table.to_arrow().sort_by("a") == expected
@@ -610,6 +678,75 @@ def test_merge_insert(db):
     assert table.to_arrow().sort_by("a") == expected
 
 
+@pytest.mark.asyncio
+async def test_merge_insert_async(db_async: AsyncConnection):
+    data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+    table = await db_async.create_table("some_table", data=data)
+    assert await table.count_rows() == 3
+    version = await table.version()
+
+    new_data = pa.table({"a": [2, 3, 4], "b": ["x", "y", "z"]})
+
+    # upsert
+    await (
+        table.merge_insert("a")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(new_data)
+    )
+    expected = pa.table({"a": [1, 2, 3, 4], "b": ["a", "x", "y", "z"]})
+    assert (await table.to_arrow()).sort_by("a") == expected
+
+    await table.checkout(version)
+    await table.restore()
+
+    # conditional update
+    await (
+        table.merge_insert("a")
+        .when_matched_update_all(where="target.b = 'b'")
+        .execute(new_data)
+    )
+    expected = pa.table({"a": [1, 2, 3], "b": ["a", "x", "c"]})
+    assert (await table.to_arrow()).sort_by("a") == expected
+
+    await table.checkout(version)
+    await table.restore()
+
+    # insert-if-not-exists
+    await table.merge_insert("a").when_not_matched_insert_all().execute(new_data)
+    expected = pa.table({"a": [1, 2, 3, 4], "b": ["a", "b", "c", "z"]})
+    assert (await table.to_arrow()).sort_by("a") == expected
+
+    await table.checkout(version)
+    await table.restore()
+
+    # replace-range
+    new_data = pa.table({"a": [2, 4], "b": ["x", "z"]})
+    await (
+        table.merge_insert("a")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .when_not_matched_by_source_delete("a > 2")
+        .execute(new_data)
+    )
+    expected = pa.table({"a": [1, 2, 4], "b": ["a", "x", "z"]})
+    assert (await table.to_arrow()).sort_by("a") == expected
+
+    await table.checkout(version)
+    await table.restore()
+
+    # replace-range no condition
+    await (
+        table.merge_insert("a")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .when_not_matched_by_source_delete()
+        .execute(new_data)
+    )
+    expected = pa.table({"a": [2, 4], "b": ["x", "z"]})
+    assert (await table.to_arrow()).sort_by("a") == expected
+
+
 def test_create_with_embedding_function(db):
     class MyTable(LanceModel):
         text: str
@@ -925,7 +1062,36 @@ def test_hybrid_search(db, tmp_path):
         .where("text='Arrrrggghhhhhhh'")
         .to_list()
     )
-    len(result) == 1
+    assert len(result) == 1
+
+    # with explicit query type
+    vector_query = list(range(emb.ndims()))
+    result = (
+        table.search(query_type="hybrid")
+        .vector(vector_query)
+        .text("Arrrrggghhhhhhh")
+        .to_arrow()
+    )
+    assert len(result) > 0
+    assert "_relevance_score" in result.column_names
+
+    # with vector_column_name
+    result = (
+        table.search(query_type="hybrid", vector_column_name="vector")
+        .vector(vector_query)
+        .text("Arrrrggghhhhhhh")
+        .to_arrow()
+    )
+    assert len(result) > 0
+    assert "_relevance_score" in result.column_names
+
+    # fail if only text or vector is provided
+    with pytest.raises(ValueError):
+        table.search(query_type="hybrid").to_list()
+    with pytest.raises(ValueError):
+        table.search(query_type="hybrid").vector(vector_query).to_list()
+    with pytest.raises(ValueError):
+        table.search(query_type="hybrid").text("Arrrrggghhhhhhh").to_list()
 
 
 @pytest.mark.parametrize(

python/python/tests/test_util.py

@@ -15,7 +15,8 @@ import os
 import pathlib
 
 import pytest
-from lancedb.util import get_uri_scheme, join_uri
+import lancedb
+from lancedb.util import get_uri_scheme, join_uri, value_to_sql
 
 
 def test_normalize_uri():
@@ -84,3 +85,29 @@ def test_local_join_uri_windows():
         assert joined == str(pathlib.Path(base) / "table.lance")
         joined = join_uri(pathlib.Path(base), "table.lance")
         assert joined == pathlib.Path(base) / "table.lance"
+
+
+def test_value_to_sql_string(tmp_path):
+    # Make sure we can convert Python string literals to SQL strings, even if
+    # they contain characters meaningful in SQL, such as ' and \.
+    values = ["anthony's", 'a "test" string', "anthony's \"favorite color\" wasn't red"]
+    expected_values = [
+        "'anthony''s'",
+        "'a \"test\" string'",
+        "'anthony''s \"favorite color\" wasn''t red'",
+    ]
+
+    for value, expected in zip(values, expected_values):
+        assert value_to_sql(value) == expected
+
+    # Also test we can roundtrip those strings through update.
+    # This validates the query parser understands the strings we
+    # are creating.
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "test",
+        [{"search": value, "replace": "something"} for value in values],
+    )
+    for value in values:
+        table.update(where=f"search = {value_to_sql(value)}", values={"replace": value})
+        assert table.to_pandas().query("search == @value")["replace"].item() == value

python/src/connection.rs

@@ -7,7 +7,7 @@ use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::From
 use lancedb::connection::{Connection as LanceConnection, CreateTableMode, LanceFileVersion};
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
-    pyclass, pyfunction, pymethods, Bound, PyAny, PyRef, PyResult, Python,
+    pyclass, pyfunction, pymethods, Bound, FromPyObject, PyAny, PyRef, PyResult, Python,
 };
 use pyo3_asyncio_0_21::tokio::future_into_py;
 
@@ -187,6 +187,7 @@ impl Connection {
 }
 
 #[pyfunction]
+#[allow(clippy::too_many_arguments)]
 pub fn connect(
     py: Python,
     uri: String,
@@ -194,6 +195,7 @@ pub fn connect(
     region: Option<String>,
     host_override: Option<String>,
     read_consistency_interval: Option<f64>,
+    client_config: Option<PyClientConfig>,
     storage_options: Option<HashMap<String, String>>,
 ) -> PyResult<Bound<'_, PyAny>> {
     future_into_py(py, async move {
@@ -214,6 +216,70 @@ pub fn connect(
         if let Some(storage_options) = storage_options {
             builder = builder.storage_options(storage_options);
         }
+        #[cfg(feature = "remote")]
+        if let Some(client_config) = client_config {
+            builder = builder.client_config(client_config.into());
+        }
         Ok(Connection::new(builder.execute().await.infer_error()?))
     })
 }
+
+#[derive(FromPyObject)]
+pub struct PyClientConfig {
+    user_agent: String,
+    retry_config: Option<PyClientRetryConfig>,
+    timeout_config: Option<PyClientTimeoutConfig>,
+}
+
+#[derive(FromPyObject)]
+pub struct PyClientRetryConfig {
+    retries: Option<u8>,
+    connect_retries: Option<u8>,
+    read_retries: Option<u8>,
+    backoff_factor: Option<f32>,
+    backoff_jitter: Option<f32>,
+    statuses: Option<Vec<u16>>,
+}
+
+#[derive(FromPyObject)]
+pub struct PyClientTimeoutConfig {
+    connect_timeout: Option<Duration>,
+    read_timeout: Option<Duration>,
+    pool_idle_timeout: Option<Duration>,
+}
+
+#[cfg(feature = "remote")]
+impl From<PyClientRetryConfig> for lancedb::remote::RetryConfig {
+    fn from(value: PyClientRetryConfig) -> Self {
+        Self {
+            retries: value.retries,
+            connect_retries: value.connect_retries,
+            read_retries: value.read_retries,
+            backoff_factor: value.backoff_factor,
+            backoff_jitter: value.backoff_jitter,
+            statuses: value.statuses,
+        }
+    }
+}
+
+#[cfg(feature = "remote")]
+impl From<PyClientTimeoutConfig> for lancedb::remote::TimeoutConfig {
+    fn from(value: PyClientTimeoutConfig) -> Self {
+        Self {
+            connect_timeout: value.connect_timeout,
+            read_timeout: value.read_timeout,
+            pool_idle_timeout: value.pool_idle_timeout,
+        }
+    }
+}
+
+#[cfg(feature = "remote")]
+impl From<PyClientConfig> for lancedb::remote::ClientConfig {
+    fn from(value: PyClientConfig) -> Self {
+        Self {
+            user_agent: value.user_agent,
+            retry_config: value.retry_config.map(Into::into).unwrap_or_default(),
+            timeout_config: value.timeout_config.map(Into::into).unwrap_or_default(),
+        }
+    }
+}