Bump version: 0.17.2-beta.2 → 0.18.0-beta.0

In the sync code example there was also an `await`.


![image](https://github.com/user-attachments/assets/4e1a1bd9-f2fb-4dbe-a9a6-1384ab63edbb)

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.14.1-beta.3"
+current_version = "0.14.2-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
+        # TODO: pass through interpreter
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
-        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -28,7 +28,7 @@ runs:
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: windows-wheels
         path: python\target\wheels

.github/workflows/make-release-commit.yml

@@ -43,7 +43,7 @@ on:
 jobs:
   make-release:
     # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
     steps:
@@ -57,15 +57,14 @@ jobs:
           # trigger any workflows watching for new tags. See:
           # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
           token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+      - name: Validate Lance dependency is at stable version
+        if: ${{ inputs.type == 'stable' }}
+        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
       - name: Bump Python version
         if: ${{ inputs.python }}
         working-directory: python
@@ -97,3 +96,7 @@ jobs:
         if: ${{ !inputs.dry_run && inputs.other }}
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        if: ${{ !inputs.dry_run && inputs.other }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/npm-publish.yml

@@ -159,7 +159,7 @@ jobs:
       - name: Install common dependencies
         run: |
           apk add protobuf-dev curl clang mold grep npm bash
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
           echo "source $HOME/.cargo/env" >> saved_env
           echo "export CC=clang" >> saved_env
           echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
@@ -167,7 +167,7 @@ jobs:
         if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
           source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
+          rustup target add aarch64-unknown-linux-musl
           crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
           sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
           apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
@@ -262,7 +262,7 @@ jobs:
       - name: Install common dependencies
         run: |
           apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
           echo "source $HOME/.cargo/env" >> saved_env
           echo "export CC=clang" >> saved_env
           echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
@@ -272,7 +272,7 @@ jobs:
         if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
           source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
+          rustup target add aarch64-unknown-linux-musl
           crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
           sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
           apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
@@ -336,7 +336,7 @@ jobs:
 
   node-windows-arm64:
     name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
-    if: startsWith(github.ref, 'refs/tags/v')
+    # if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     container: alpine:edge
     strategy:
@@ -351,12 +351,12 @@ jobs:
       - name: Install dependencies
         run: |
           apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
           echo "source $HOME/.cargo/env" >> saved_env
           echo "export CC=clang" >> saved_env
           echo "export AR=llvm-ar" >> saved_env
           source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
           (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
           echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
           echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
@@ -416,7 +416,7 @@ jobs:
   nodejs-windows-arm64:
     name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
     # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
+    # if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     container: alpine:edge
     strategy:
@@ -431,12 +431,12 @@ jobs:
       - name: Install dependencies
         run: |
           apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
           echo "source $HOME/.cargo/env" >> saved_env
           echo "export CC=clang" >> saved_env
           echo "export AR=llvm-ar" >> saved_env
           source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc --toolchain 1.80.0
+          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
           (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
           echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
           echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
@@ -571,7 +571,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
-          persist-credentials: false
+          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
           fetch-depth: 0
           lfs: true
       - uses: ./.github/workflows/update_package_lock
@@ -589,7 +589,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: main
-          persist-credentials: false
+          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
           fetch-depth: 0
           lfs: true
       - uses: ./.github/workflows/update_package_lock_nodejs

.github/workflows/python.yml

@@ -30,10 +30,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install ruff
         run: |
-          pip install ruff==0.5.4
+          pip install ruff==0.8.4
       - name: Format check
         run: ruff format --check .
       - name: Lint

.github/workflows/rust.yml

@@ -185,7 +185,7 @@ jobs:
           Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
 
           # Add MSVC runtime libraries to LIB
-          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
+          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
                      "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
                      "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
           Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
@@ -238,3 +238,41 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build --target aarch64-pc-windows-msvc
           cargo test --target aarch64-pc-windows-msvc
+
+  msrv:
+    # Check the minimum supported Rust version
+    name: MSRV Check - Rust v${{ matrix.msrv }}
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install ${{ matrix.msrv }}
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ matrix.msrv }}
+      - name: Downgrade  dependencies
+        # These packages have newer requirements for MSRV
+        run: |
+          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
+          cargo update -p aws-sdk-dynamodb --precise 1.55.0
+          cargo update -p aws-config --precise 1.5.10
+          cargo update -p aws-sdk-kms --precise 1.51.0
+          cargo update -p aws-sdk-s3 --precise 1.65.0
+          cargo update -p aws-sdk-sso --precise 1.50.0
+          cargo update -p aws-sdk-ssooidc --precise 1.51.0
+          cargo update -p aws-sdk-sts --precise 1.51.0
+          cargo update -p home --precise 0.5.9
+      - name: cargo +${{ matrix.msrv }} check
+        run: cargo check --workspace --tests --benches --all-features

.github/workflows/upload_wheel/action.yml

@@ -22,7 +22,7 @@ runs:
     shell: bash
     id: choose_repo
     run: |
-      if [ ${{ github.ref }} == "*beta*" ]; then
+      if [[ ${{ github.ref }} == *beta* ]]; then
         echo "repo=fury" >> $GITHUB_OUTPUT
       else
         echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -33,7 +33,7 @@ runs:
       FURY_TOKEN: ${{ inputs.fury_token }}
       PYPI_TOKEN: ${{ inputs.pypi_token }}
     run: |
-      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
+      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
         WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
         echo "Uploading $WHEEL to Fury"
         curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/

CONTRIBUTING.md

@@ -0,0 +1,78 @@
+# Contributing to LanceDB
+
+LanceDB is an open-source project and we welcome contributions from the community.
+This document outlines the process for contributing to LanceDB.
+
+## Reporting Issues
+
+If you encounter a bug or have a feature request, please open an issue on the
+[GitHub issue tracker](https://github.com/lancedb/lancedb).
+
+## Picking an issue
+
+We track issues on the GitHub issue tracker. If you are looking for something to
+work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
+
+If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
+
+## Configuring Git
+
+First, fork the repository on GitHub, then clone your fork:
+
+```bash
+git clone https://github.com/<username>/lancedb.git
+cd lancedb
+```
+
+Then add the main repository as a remote:
+
+```bash
+git remote add upstream https://github.com/lancedb/lancedb.git
+git fetch upstream
+```
+
+## Setting up your development environment
+
+We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
+
+* [Python](python/CONTRIBUTING.md)
+* [Typescript](nodejs/CONTRIBUTING.md)
+<!-- TODO: add Java contributing guide -->
+* [Documentation](docs/README.md)
+
+
+## Best practices for pull requests
+
+For the best chance of having your pull request accepted, please follow these guidelines:
+
+1. Unit test all bug fixes and new features. Your code will not be merged if it
+   doesn't have tests.
+1. If you change the public API, update the documentation in the `docs` directory.
+1. Aim to minimize the number of changes in each pull request. Keep to solving
+   one problem at a time, when possible.
+1. Before marking a pull request ready-for-review, do a self review of your code.
+   Is it clear why you are making the changes? Are the changes easy to understand?
+1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
+    * New feature: `feat: adding foo API`
+    * Bug fix: `fix: issue with foo API`
+    * Documentation change: `docs: adding foo API documentation`
+1. If your pull request is a work in progress, leave the pull request as a draft.
+   We will assume the pull request is ready for review when it is opened.
+1. When writing tests, test the error cases. Make sure they have understandable
+   error messages.
+
+## Project structure
+
+The core library is written in Rust. The Python, Typescript, and Java libraries
+are wrappers around the Rust library.
+
+* `src/lancedb`: Rust library source code
+* `python`: Python package source code
+* `nodejs`: Typescript package source code
+* `node`: **Deprecated** Typescript package source code
+* `java`: Java package source code
+* `docs`: Documentation source code
+
+## Release process
+
+For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -18,19 +18,19 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.80.0"                                                     # TODO: lower this once we upgrade Lance again.
+rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.21.0", "features" = [
+lance = { "version" = "=0.21.1", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-io = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-index = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-linalg = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-table = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-testing = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-datafusion = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
-lance-encoding = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.3" }
+], git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-io = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-index = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-linalg = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-table = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-testing = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-datafusion = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
+lance-encoding = { version = "=0.21.1", git = "https://github.com/lancedb/lance.git", tag = "v0.21.1-beta.2" }
 # Note that this one does not include pyarrow
 arrow = { version = "53.2", optional = false }
 arrow-array = "53.2"

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -53,7 +53,7 @@ curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-42
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
 
-# fwpuclnt.lib arm64rt.lib
+# dbghelp.lib fwpuclnt.lib arm64rt.lib
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
 
@@ -98,7 +98,7 @@ find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#inclu
 # reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
 # I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
 
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
 
 (cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
 

ci/validate_stable_lance.py

@@ -0,0 +1,34 @@
+import tomllib
+
+found_preview_lance = False
+
+with open("Cargo.toml", "rb") as f:
+    cargo_data = tomllib.load(f)
+
+    for name, dep in cargo_data["workspace"]["dependencies"].items():
+        if name == "lance" or name.startswith("lance-"):
+            if isinstance(dep, str):
+                version = dep
+            elif isinstance(dep, dict):
+                # Version doesn't have the beta tag in it, so we instead look
+                # at the git tag.
+                version = dep["tag"]
+            else:
+                raise ValueError("Unexpected type for dependency: " + str(dep))
+
+            if "beta" in version:
+                found_preview_lance = True
+                print(f"Dependency '{name}' is a preview version: {version}")
+
+with open("python/pyproject.toml", "rb") as f:
+    py_proj_data = tomllib.load(f)
+
+    for dep in py_proj_data["project"]["dependencies"]:
+        if dep.startswith("pylance"):
+            if "b" in dep:
+                found_preview_lance = True
+                print(f"Dependency '{dep}' is a preview version")
+            break  # Only one pylance dependency
+
+if found_preview_lance:
+    raise ValueError("Found preview version of Lance in dependencies")

docs/README.md

@@ -9,36 +9,81 @@ unreleased features.
 ## Building the docs
 
 ### Setup
-1. Install LanceDB. From LanceDB repo root: `pip install -e python`
-2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
-3. Make sure you have node and npm setup
-4. Make sure protobuf and libssl are installed
+1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
+   Run `make develop` to install the Python package.
+2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
 
-### Building node module and create markdown files
+### Preview the docs
 
-See [Javascript docs README](./src/javascript/README.md)
-
-### Build docs
-From LanceDB repo root:
-
-Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-### Run local server
-
-You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
-
-```bash
+```shell
 cd docs
 mkdocs serve
 ```
 
-### Run doctest for typescript example
+If you want to just generate the HTML files:
 
-```bash
-cd lancedb/docs
-npm i
-npm run build
-npm run all
+```shell
+PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
+```
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+## Adding examples
+
+To make sure examples are correct, we put examples in test files so they can be
+run as part of our test suites.
+
+You can see the tests are at:
+
+* Python: `python/python/tests/docs`
+* Typescript: `nodejs/examples/`
+
+### Checking python examples
+
+```shell
+cd python
+pytest -vv python/tests/docs
+```
+
+### Checking typescript examples
+
+The `@lancedb/lancedb` package must be built before running the tests:
+
+```shell
+pushd nodejs
+npm ci
+npm run build
+popd
+```
+
+Then you can run the examples by going to the `nodejs/examples` directory and
+running the tests like a normal npm package:
+
+```shell
+pushd nodejs/examples
+npm ci
+npm test
+popd
+```
+
+## API documentation
+
+### Python
+
+The Python API documentation is organized based on the file `docs/src/python/python.md`.
+We manually add entries there so we can control the organization of the reference page.
+**However, this means any new types must be manually added to the file.** No additional
+steps are needed to generate the API documentation.
+
+### Typescript
+
+The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
+
+When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
+The new files should be checked into the repository.
+
+```shell
+pushd nodejs
+npm run docs
+popd
 ```

docs/mkdocs.yml

@@ -62,6 +62,7 @@ plugins:
             # for cross references
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
+            - https://lancedb.github.io/lance/objects.inv
   - mkdocs-jupyter
   - render_swagger:
       allow_arbitrary_locations: true
@@ -145,7 +146,9 @@ nav:
               - Building Custom Rerankers: reranking/custom_reranker.md
               - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
-          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+          - Versioning & Reproducibility: 
+              - sync API: notebooks/reproducibility.ipynb
+              - async API: notebooks/reproducibility_async.ipynb
           - Configuring Storage: guides/storage.md
           - Migration Guide: migration.md
           - Tuning retrieval performance:
@@ -277,7 +280,9 @@ nav:
           - Building Custom Rerankers: reranking/custom_reranker.md
           - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
-      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+      - Versioning & Reproducibility: 
+          - sync API: notebooks/reproducibility.ipynb
+          - async API: notebooks/reproducibility_async.ipynb
       - Configuring Storage: guides/storage.md
       - Migration Guide: migration.md
       - Tuning retrieval performance:

docs/src/ann_indexes.md

@@ -18,25 +18,24 @@ See the [indexing](concepts/index_ivfpq.md) concepts guide for more information
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
+    === "Sync API"
 
-    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-    ```python
-    import lancedb
-    import numpy as np
-    uri = "data/sample-lancedb"
-    db = lancedb.connect(uri)
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
+        ```
+    === "Async API"
+        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-    # Create 10,000 sample vectors
-    data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
-
-    # Add the vectors to a table
-    tbl = db.create_table("my_vectors", data=data)
-
-    # Create and train the index - you need to have enough data in the table for an effective training step
-    tbl.create_index(num_partitions=256, num_sub_vectors=96)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
+        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
+        ```
 
 === "TypeScript"
 
@@ -127,7 +126,9 @@ You can specify the GPU device to train IVF partitions via
         accelerator="mps"
     )
     ```
-
+!!! note
+    GPU based indexing is not yet supported with our asynchronous client.
+    
 Troubleshooting:
 
 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
@@ -152,14 +153,16 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 
 === "Python"
+    === "Sync API"
 
-    ```python
-    tbl.search(np.random.random((1536))) \
-        .limit(2) \
-        .nprobes(20) \
-        .refine_factor(10) \
-        .to_pandas()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
+        ```
 
     ```text
                                               vector       item       _distance
@@ -196,10 +199,16 @@ The search will return the data requested in addition to the distance of each it
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
+    === "Sync API"
 
-    ```python
-    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
+        ```
 
 === "TypeScript"
 
@@ -221,10 +230,16 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-    ```python
-    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-    ```
+    === "Sync API"
 
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
+        ```
 
     ```text
                                                 vector _distance

docs/src/basic.md

@@ -141,14 +141,6 @@ recommend switching to stable releases.
     --8<-- "python/python/tests/docs/test_basic.py:connect_async"
     ```
 
-    !!! note "Asynchronous Python API"
-
-        The asynchronous Python API is new and has some slight differences compared
-        to the synchronous API.  Feel free to start using the asynchronous version.
-        Once all features have migrated we will start to move the synchronous API to
-        use the same syntax as the asynchronous API.  To help with this migration we
-        have created a [migration guide](migration.md) detailing the differences.
-
 === "Typescript[^1]"
 
     === "@lancedb/lancedb"

docs/src/fts.md

@@ -10,28 +10,20 @@ LanceDB provides support for full-text search via Lance, allowing you to incorpo
 Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
 
 === "Python"
+    === "Sync API"
 
-    ```python
-    import lancedb
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
+        ```
+    === "Async API"
 
-    uri = "data/sample-lancedb"
-    db = lancedb.connect(uri)
-
-    table = db.create_table(
-        "my_table",
-        data=[
-            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
-        ],
-    )
-
-    # passing `use_tantivy=False` to use lance FTS index
-    # `use_tantivy=True` by default
-    table.create_fts_index("text", use_tantivy=False)
-    table.search("puppy").limit(10).select(["text"]).to_list()
-    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
-    # ...
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
+        ```
 
 === "TypeScript"
 
@@ -50,7 +42,7 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
     });
 
     await tbl
-        .search("puppy", queryType="fts")
+        .search("puppy", "fts")
         .select(["text"])
         .limit(10)
         .toArray();
@@ -93,22 +85,32 @@ By default the text is tokenized by splitting on punctuation and whitespaces, an
 Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.
 
 For example, to enable stemming for English:
-```python
-table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
-```
+=== "Sync API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
+    ```
 
 the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
 
 The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.
 
 For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-```python
-table.create_fts_index("text",
-                        use_tantivy=False,
-                        language="French",
-                        stem=True,
-                        ascii_folding=True)
-```
+=== "Sync API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
+    ```
 
 ## Filtering
 
@@ -119,9 +121,16 @@ This can be invoked via the familiar `where` syntax.
 With pre-filtering:
 === "Python"
 
-    ```python
-    table.search("puppy").limit(10).where("meta='foo'", prefilte=True).to_list()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
+        ```
 
 === "TypeScript"
 
@@ -151,9 +160,16 @@ With pre-filtering:
 With post-filtering:
 === "Python"
 
-    ```python
-    table.search("puppy").limit(10).where("meta='foo'", prefilte=False).to_list()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
+        ```
 
 === "TypeScript"
 
@@ -191,9 +207,16 @@ or a **terms** search query like `old man sea`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
 
 To search for a phrase, the index must be created with `with_position=True`:
-```python
-table.create_fts_index("text", use_tantivy=False, with_position=True)
-```
+=== "Sync API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
+    ```
 This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
 
 
@@ -205,10 +228,16 @@ This can make the query more efficient, especially when the table is large and t
 
 === "Python"
 
-    ```python
-    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
-    table.optimize()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
+        ```
 
 === "TypeScript"
 

docs/src/fts_tantivy.md

@@ -2,7 +2,7 @@
 
 LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
 
-The tantivy-based FTS is only available in Python and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
+The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
 
 ## Installation
 

docs/src/guides/scalar_index.md

@@ -32,19 +32,20 @@ over scalar columns.
 ### Create a scalar index
 === "Python"
 
-    ```python
-    import lancedb
-    books = [
-      {"book_id": 1, "publisher": "plenty of books", "tags": ["fantasy", "adventure"]},
-      {"book_id": 2, "publisher": "book town", "tags": ["non-fiction"]},
-      {"book_id": 3, "publisher": "oreilly", "tags": ["textbook"]}
-    ]
+    === "Sync API"
 
-    db = lancedb.connect("./db")
-    table = db.create_table("books", books)
-    table.create_scalar_index("book_id")  # BTree by default
-    table.create_scalar_index("publisher", index_type="BITMAP")
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
+        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
+        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
+        ```
 
 === "Typescript"
 
@@ -62,12 +63,18 @@ The following scan will be faster if the column `book_id` has a scalar index:
 
 === "Python"
 
-    ```python
-    import lancedb
+    === "Sync API"
 
-    table = db.open_table("books")
-    my_df = table.search().where("book_id = 2").to_pandas()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
+        ```
 
 === "Typescript"
 
@@ -88,22 +95,18 @@ Scalar indices can also speed up scans containing a vector search or full text s
 
 === "Python"
 
-    ```python
-    import lancedb
+    === "Sync API"
 
-    data = [
-      {"book_id": 1, "vector": [1, 2]},
-      {"book_id": 2, "vector": [3, 4]},
-      {"book_id": 3, "vector": [5, 6]}
-    ]
-    table = db.create_table("book_with_embeddings", data)
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
+        ```
+    === "Async API"
 
-    (
-        table.search([1, 2])
-        .where("book_id != 3", prefilter=True)
-        .to_pandas()
-    )
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
+        ```
 
 === "Typescript"
 
@@ -122,10 +125,16 @@ Scalar indices can also speed up scans containing a vector search or full text s
 Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
 === "Python"
 
-    ```python
-    table.add([{"vector": [7, 8], "book_id": 4}])
-    table.optimize()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
+        ```
 
 === "TypeScript"
 

docs/src/guides/storage.md

@@ -12,26 +12,50 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 === "Python"
 
     AWS S3:
+    === "Sync API"
 
-    ```python
-    import lancedb
-    db = lancedb.connect("s3://bucket/path")
-    ```
+        ```python
+        import lancedb
+        db = lancedb.connect("s3://bucket/path")
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("s3://bucket/path")
+        ```
 
     Google Cloud Storage:
 
-    ```python
-    import lancedb
-    db = lancedb.connect("gs://bucket/path")
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect("gs://bucket/path")
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("gs://bucket/path")
+        ```
 
     Azure Blob Storage:
 
     <!-- skip-test -->
-    ```python
-    import lancedb
-    db = lancedb.connect("az://bucket/path")
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect("az://bucket/path")
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("az://bucket/path")
+        ```
     Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
 
 
@@ -94,13 +118,24 @@ If you only want this to apply to one particular connection, you can pass the `s
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={"timeout": "60s"}
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "s3://bucket/path",
+            storage_options={"timeout": "60s"}
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={"timeout": "60s"}
+        )
+        ```
 
 === "TypeScript"
 
@@ -128,15 +163,29 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
 === "Python"
 
     <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async("s3://bucket/path")
-    table = await db.create_table(
-        "table",
-        [{"a": 1, "b": 2}],
-        storage_options={"timeout": "60s"}
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect("s3://bucket/path")
+        table = db.create_table(
+            "table",
+            [{"a": 1, "b": 2}],
+            storage_options={"timeout": "60s"}
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async("s3://bucket/path")
+        async_table = await async_db.create_table(
+            "table",
+            [{"a": 1, "b": 2}],
+            storage_options={"timeout": "60s"}
+        )
+        ```
 
 === "TypeScript"
 
@@ -194,17 +243,32 @@ These can be set as environment variables or passed in the `storage_options` par
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "aws_access_key_id": "my-access-key",
-            "aws_secret_access_key": "my-secret-key",
-            "aws_session_token": "my-session-token",
-        }
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "s3://bucket/path",
+            storage_options={
+                "aws_access_key_id": "my-access-key",
+                "aws_secret_access_key": "my-secret-key",
+                "aws_session_token": "my-session-token",
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={
+                "aws_access_key_id": "my-access-key",
+                "aws_secret_access_key": "my-secret-key",
+                "aws_session_token": "my-session-token",
+            }
+        )
+        ```
 
 === "TypeScript"
 
@@ -348,12 +412,22 @@ name of the table to use.
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+        )    
+        ```
 
 === "JavaScript"
 
@@ -441,16 +515,30 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "region": "us-east-1",
-            "endpoint": "http://minio:9000",
-        }
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "s3://bucket/path",
+            storage_options={
+                "region": "us-east-1",
+                "endpoint": "http://minio:9000",
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://bucket/path",
+            storage_options={
+                "region": "us-east-1",
+                "endpoint": "http://minio:9000",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -502,16 +590,30 @@ To configure LanceDB to use an S3 Express endpoint, you must set the storage opt
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://my-bucket--use1-az4--x-s3/path",
-        storage_options={
-            "region": "us-east-1",
-            "s3_express": "true",
-        }
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            storage_options={
+                "region": "us-east-1",
+                "s3_express": "true",
+            }
+        )
+        ```
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            storage_options={
+                "region": "us-east-1",
+                "s3_express": "true",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -552,15 +654,29 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
 === "Python"
 
     <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "gs://my-bucket/my-database",
-        storage_options={
-            "service_account": "path/to/service-account.json",
-        }
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "gs://my-bucket/my-database",
+            storage_options={
+                "service_account": "path/to/service-account.json",
+            }
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "gs://my-bucket/my-database",
+            storage_options={
+                "service_account": "path/to/service-account.json",
+            }
+        )    
+        ```
 
 === "TypeScript"
 
@@ -612,16 +728,31 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
 === "Python"
 
     <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "az://my-container/my-database",
-        storage_options={
-            account_name: "some-account",
-            account_key: "some-key",
-        }
-    )
-    ```
+    === "Sync API"
+
+        ```python
+        import lancedb
+        db = lancedb.connect(
+            "az://my-container/my-database",
+            storage_options={
+                account_name: "some-account",
+                account_key: "some-key",
+            }
+        )
+        ```
+    <!-- skip-test -->
+    === "Async API"
+
+        ```python
+        import lancedb
+        async_db = await lancedb.connect_async(
+            "az://my-container/my-database",
+            storage_options={
+                account_name: "some-account",
+                account_key: "some-key",
+            }
+        )   
+        ```
 
 === "TypeScript"
 

docs/src/guides/tables.md

@@ -12,10 +12,18 @@ Initialize a LanceDB connection and create a table
 
 === "Python"
 
-    ```python
-    import lancedb
-    db = lancedb.connect("./.lancedb")
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:connect"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:connect_async"
+        ```
 
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
@@ -47,18 +55,16 @@ Initialize a LanceDB connection and create a table
 
 === "Python"
 
-    ```python
-    import lancedb
+    === "Sync API"
 
-    db = lancedb.connect("./.lancedb")
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table"
+        ```
+    === "Async API"
 
-    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
-            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
-
-    db.create_table("my_table", data)
-
-    db["my_table"].head()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async"
+        ```
 
     !!! info "Note"
         If the table already exists, LanceDB will raise an error by default.
@@ -67,16 +73,30 @@ Initialize a LanceDB connection and create a table
         and the table exists, then it simply opens the existing table. The data you
         passed in will NOT be appended to the table in that case.
 
-    ```python
-    db.create_table("name", data, exist_ok=True)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_exist_ok"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_exist_ok"
+        ```
 
     Sometimes you want to make sure that you start fresh. If you want to
     overwrite the table, you can pass in mode="overwrite" to the createTable function.
 
-    ```python
-    db.create_table("name", data, mode="overwrite")
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_overwrite"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_overwrite"
+        ```
 
 === "Typescript[^1]"
     You can create a LanceDB table in JavaScript using an array of records as follows.
@@ -146,34 +166,37 @@ Initialize a LanceDB connection and create a table
 
 ### From a Pandas DataFrame
 
-```python
-import pandas as pd
 
-data = pd.DataFrame({
-    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
-    "lat": [45.5, 40.1],
-    "long": [-122.7, -74.1]
-})
+=== "Sync API"
 
-db.create_table("my_table", data)
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pandas"
+    ```
+=== "Async API"
 
-db["my_table"].head()
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pandas"
+    ```
 
 !!! info "Note"
     Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
 The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.
 
-```python
-custom_schema = pa.schema([
-pa.field("vector", pa.list_(pa.float32(), 4)),
-pa.field("lat", pa.float32()),
-pa.field("long", pa.float32())
-])
+=== "Sync API"
 
-table = db.create_table("my_table", data, schema=custom_schema)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_custom_schema"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_custom_schema"
+    ```
 
 ### From a Polars DataFrame
 
@@ -182,45 +205,38 @@ written in Rust. Just like in Pandas, the Polars integration is enabled by PyArr
 under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
 is on the way.
 
-```python
-import polars as pl
+=== "Sync API"
 
-data = pl.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pl_table", data=data)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_polars"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_polars"
+    ```
 
 ### From an Arrow Table
 You can also create LanceDB tables directly from Arrow tables.
 LanceDB supports float16 data type!
 
 === "Python"
+    === "Sync API"
 
-    ```python
-    import pyarrows as pa
-    import numpy as np
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_arrow_table"
+        ```
+    === "Async API"
 
-    dim = 16
-    total = 2
-    schema = pa.schema(
-        [
-            pa.field("vector", pa.list_(pa.float16(), dim)),
-            pa.field("text", pa.string())
-        ]
-    )
-    data = pa.Table.from_arrays(
-        [
-            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
-                    pa.list_(pa.float16(), dim)),
-            pa.array(["foo", "bar"])
-        ],
-        ["vector", "text"],
-    )
-    tbl = db.create_table("f16_tbl", data, schema=schema)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_arrow_table"
+        ```
 
 === "Typescript[^1]"
 
@@ -250,25 +266,22 @@ can be configured with the vector dimensions. It is also important to note that
 LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
 (which itself derives from `pydantic.BaseModel`).
 
-```python
-from lancedb.pydantic import Vector, LanceModel
+=== "Sync API"
 
-class Content(LanceModel):
-    movie_id: int
-    vector: Vector(128)
-    genres: str
-    title: str
-    imdb_id: int
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pydantic"
+    ```
+=== "Async API"
 
-    @property
-    def imdb_url(self) -> str:
-        return f"https://www.imdb.com/title/tt{self.imdb_id}"
-
-import pyarrow as pa
-db = lancedb.connect("~/.lancedb")
-table_name = "movielens_small"
-table = db.create_table(table_name, schema=Content)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pydantic"
+    ```
 
 #### Nested schemas
 
@@ -277,22 +290,24 @@ For example, you may want to store the document string
 and the document source name as a nested Document object:
 
 ```python
-class Document(BaseModel):
-    content: str
-    source: str
+--8<-- "python/python/tests/docs/test_guide_tables.py:import-pydantic-basemodel"
+--8<-- "python/python/tests/docs/test_guide_tables.py:class-Document"
 ```
 
 This can be used as the type of a LanceDB table column:
 
-```python
-class NestedSchema(LanceModel):
-    id: str
-    vector: Vector(1536)
-    document: Document
+=== "Sync API"
 
-tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_nested_schema"
+    ```
+=== "Async API"
 
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_nested_schema"
+    ```
 This creates a struct column called "document" that has two subfields
 called "content" and "source":
 
@@ -356,29 +371,20 @@ LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generat
 
 Here's an example using using `RecordBatch` iterator for creating tables.
 
-```python
-import pyarrow as pa
+=== "Sync API"
 
-def make_batches():
-    for i in range(5):
-        yield pa.RecordBatch.from_arrays(
-            [
-                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-                        pa.list_(pa.float32(), 4)),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"],
-        )
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_batch"
+    ```
+=== "Async API"
 
-schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 4)),
-    pa.field("item", pa.utf8()),
-    pa.field("price", pa.float32()),
-])
-
-db.create_table("batched_tale", make_batches(), schema=schema)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
+    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_batch"
+    ```
 
 You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
@@ -387,15 +393,29 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 === "Python"
     If you forget the name of your table, you can always get a listing of all table names.
 
-    ```python
-    print(db.table_names())
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables_async"
+        ```
 
     Then, you can open any existing tables.
 
-    ```python
-    tbl = db.open_table("my_table")
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table_async"
+        ```
 
 === "Typescript[^1]"
 
@@ -418,35 +438,41 @@ You can create an empty table for scenarios where you want to add data to the ta
 
 
     An empty table can be initialized via a PyArrow schema.
+    === "Sync API"
 
-    ```python
-    import lancedb
-    import pyarrow as pa
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table"
+        ```
+    === "Async API"
 
-    schema = pa.schema(
-      [
-          pa.field("vector", pa.list_(pa.float32(), 2)),
-          pa.field("item", pa.string()),
-          pa.field("price", pa.float32()),
-      ])
-    tbl = db.create_table("empty_table_add", schema=schema)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async"
+        ```
 
     Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
     directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
     that has been extended to support LanceDB specific types like `Vector`.
 
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, vector
+    === "Sync API"
 
-    class Item(LanceModel):
-        vector: Vector(2)
-        item: str
-        price: float
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_pydantic"
+        ```
+    === "Async API"
 
-    tbl = db.create_table("empty_table_add", schema=Item.to_arrow_schema())
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async_pydantic"
+        ```
 
     Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
@@ -473,86 +499,96 @@ After a table has been created, you can always add more data to it using the `ad
 
     ### Add a Pandas DataFrame
 
-    ```python
-    df = pd.DataFrame({
-        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
-    })
-    tbl.add(df)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pandas"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pandas"
+        ```
 
     ### Add a Polars DataFrame
 
-    ```python
-    df = pl.DataFrame({
-        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
-    })
-    tbl.add(df)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_polars"
+        ```
+    === "Async API"
+    
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_polars"
+        ```
 
     ### Add an Iterator
 
     You can also add a large dataset batch in one go using Iterator of any supported data types.
 
-    ```python
-    def make_batches():
-        for i in range(5):
-            yield [
-                    {"vector": [3.1, 4.1], "item": "peach", "price": 6.0},
-                    {"vector": [5.9, 26.5], "item": "pear", "price": 5.0}
-                ]
-    tbl.add(make_batches())
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_batch"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_batch"
+        ```
 
     ### Add a PyArrow table
 
     If you have data coming in as a PyArrow table, you can add it directly to the LanceDB table.
 
-    ```python
-    pa_table = pa.Table.from_arrays(
-            [
-                pa.array([[9.1, 6.7], [9.9, 31.2]],
-                        pa.list_(pa.float32(), 2)),
-                pa.array(["mango", "orange"]),
-                pa.array([7.0, 4.0]),
-            ],
-            ["vector", "item", "price"],
-        )
+    === "Sync API"
 
-    tbl.add(pa_table)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pyarrow"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pyarrow"
+        ```
 
     ### Add a Pydantic Model
 
     Assuming that a table has been created with the correct schema as shown [above](#creating-empty-table), you can add data items that are valid Pydantic models to the table.
 
-    ```python
-    pydantic_model_items = [
-        Item(vector=[8.1, 4.7], item="pineapple", price=10.0),
-        Item(vector=[6.9, 9.3], item="avocado", price=9.0)
-    ]
+    === "Sync API"
 
-    tbl.add(pydantic_model_items)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pydantic"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pydantic"
+        ```
 
     ??? "Ingesting Pydantic models with LanceDB embedding API"
         When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` field as None to allow LanceDB to automatically vectorize the data.
 
-        ```python
-        import lancedb
-        from lancedb.pydantic import LanceModel, Vector
-        from lancedb.embeddings import get_registry
+        === "Sync API"
 
-        db = lancedb.connect("~/tmp")
-        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
+            ```python
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_with_embedding"
+            ```
+        === "Async API"
 
-        class Schema(LanceModel):
-            text: str = embed_fcn.SourceField()
-            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
-
-        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
-        models = [Schema(text="hello"), Schema(text="world")]
-        tbl.add(models)
-        ```
+            ```python
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
+            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_with_embedding"
+            ```
 
 === "Typescript[^1]"
 
@@ -571,44 +607,41 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
 === "Python"
 
-    ```python
-    tbl.delete('item = "fizz"')
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row_async"
+        ```
 
     ### Deleting row with specific column value
 
-    ```python
-    import lancedb
+    === "Sync API"
 
-    data = [{"x": 1, "vector": [1, 2]},
-            {"x": 2, "vector": [3, 4]},
-            {"x": 3, "vector": [5, 6]}]
-    db = lancedb.connect("./.lancedb")
-    table = db.create_table("my_table", data)
-    table.to_pandas()
-    #   x      vector
-    # 0  1  [1.0, 2.0]
-    # 1  2  [3.0, 4.0]
-    # 2  3  [5.0, 6.0]
-
-    table.delete("x = 2")
-    table.to_pandas()
-    #   x      vector
-    # 0  1  [1.0, 2.0]
-    # 1  3  [5.0, 6.0]
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row"
+        ```
+    === "Async API"
 
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row_async"
+        ```
+    
     ### Delete from a list of values
+    === "Sync API"
 
-    ```python
-    to_remove = [1, 5]
-    to_remove = ", ".join(str(v) for v in to_remove)
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values"
+        ```
+    === "Async API"
 
-    table.delete(f"x IN ({to_remove})")
-    table.to_pandas()
-    #   x      vector
-    # 0  3  [5.0, 6.0]
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values_async"
+        ```
 
 === "Typescript[^1]"
 
@@ -659,27 +692,20 @@ This can be used to update zero to all rows depending on how many rows match the
 === "Python"
 
     API Reference: [lancedb.table.Table.update][]
+    === "Sync API"
 
-    ```python
-    import lancedb
-    import pandas as pd
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table"
+        ```
+    === "Async API"
 
-    # Create a lancedb connection
-    db = lancedb.connect("./.lancedb")
-
-    # Create a table from a pandas DataFrame
-    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-    table = db.create_table("my_table", data)
-
-    # Update the table where x = 2
-    table.update(where="x = 2", values={"vector": [10, 10]})
-
-    # Get the updated table as a pandas DataFrame
-    df = table.to_pandas()
-
-    # Print the DataFrame
-    print(df)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_async"
+        ```
 
     Output
     ```shell
@@ -734,13 +760,16 @@ This can be used to update zero to all rows depending on how many rows match the
   The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
 
 === "Python"
+    === "Sync API"
 
-    ```python
-    # Update the table where x = 2
-    table.update(valuesSql={"x": "x + 1"})
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql"
+        ```
+    === "Async API"
 
-    print(table.to_pandas())
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql_async"
+        ```
 
     Output
     ```shell
@@ -771,11 +800,16 @@ This can be used to update zero to all rows depending on how many rows match the
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
+    === "Sync API"
 
-      ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-      ```
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+        ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
       By default, if the table does not exist an exception is raised. To suppress this,
@@ -809,9 +843,16 @@ data type for it.
 
 === "Python"
 
-    ```python
-    --8<-- "python/python/tests/docs/test_basic.py:add_columns"
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:add_columns"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:add_columns_async"
+        ```
     **API Reference:** [lancedb.table.Table.add_columns][]
 
 === "Typescript"
@@ -848,10 +889,18 @@ rewriting the column, which can be a heavy operation.
 
 === "Python"
 
-    ```python
-    import pyarrow as pa
-    --8<-- "python/python/tests/docs/test_basic.py:alter_columns"
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+        --8<-- "python/python/tests/docs/test_basic.py:alter_columns"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
+        --8<-- "python/python/tests/docs/test_basic.py:alter_columns_async"
+        ```
     **API Reference:** [lancedb.table.Table.alter_columns][]
 
 === "Typescript"
@@ -872,9 +921,16 @@ will remove the column from the schema.
 
 === "Python"
 
-    ```python
-     --8<-- "python/python/tests/docs/test_basic.py:drop_columns"
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:drop_columns"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_basic.py:drop_columns_async"
+        ```
     **API Reference:** [lancedb.table.Table.drop_columns][]
 
 === "Typescript"
@@ -925,31 +981,46 @@ There are three possible settings for `read_consistency_interval`:
 
     To set strong consistency, use `timedelta(0)`:
 
-    ```python
-    from datetime import timedelta
-    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
-    table = db.open_table("my_table")
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_strong_consistency"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_strong_consistency"
+        ```
 
     For eventual consistency, use a custom `timedelta`:
 
-    ```python
-    from datetime import timedelta
-    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
-    table = db.open_table("my_table")
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_eventual_consistency"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_eventual_consistency"
+        ```
 
     By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
 
-    ```python
-    db = lancedb.connect("./.lancedb")
-    table = db.open_table("my_table")
+    === "Sync API"
 
-    # (Other writes happen to my_table from another process)
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_checkout_latest"
+        ```
+    === "Async API"
 
-    # Check for updates
-    table.checkout_latest()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_checkout_latest"
+        ```
 
 === "Typescript[^1]"
 
@@ -957,14 +1028,14 @@ There are three possible settings for `read_consistency_interval`:
 
     ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
-    const table = await db.openTable("my_table");
+    const tbl = await db.openTable("my_table");
     ```
 
     For eventual consistency, specify the update interval as seconds:
 
     ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
-    const table = await db.openTable("my_table");
+    const tbl = await db.openTable("my_table");
     ```
 
 <!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,8 +1,8 @@
 ## Improving retriever performance
 
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
-VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
 
 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
 
@@ -19,7 +19,7 @@ Using different embedding models is something that's very specific to the use ca
 
 
 ## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
 
 ### Using different query types
 Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
@@ -45,14 +45,14 @@ table.add(df[["context"]].to_dict(orient="records"))
 queries = df["query"].tolist()
 ```
 
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
 
 * <b> Vector Search: </b>
 
     ```python
     table.search(quries[0], query_type="vector").limit(5).to_pandas()
     ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
 
     ```python
     table.search(quries[0]).limit(5).to_pandas()
@@ -77,7 +77,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
 * <b> Hybrid Search: </b>
 
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
     ```python
     table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
     ```
@@ -87,7 +87,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
     !!! note "Note"
         By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
 
     
 

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,6 +1,6 @@
 Continuing from the previous section, we can now rerank the results using more complex rerankers.
 
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:
@@ -62,9 +62,6 @@ Let us take a look at the same datasets from the previous sections, using the sa
 | Reranked fts  | 0.672    |
 | Hybrid | 0.759 |
 
-### SQuAD Dataset
-
-
 ### Uber10K sec filing Dataset
 
 | Query Type | Hit-rate@5 |

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -1,5 +1,5 @@
 ## Finetuning the Embedding Model
-Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
 
@@ -16,7 +16,7 @@ validation_df.to_csv("data_val.csv", index=False)
 You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
 
 
-Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
+We parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node:
 ```python
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.readers.file import PagedCSVReader
@@ -43,7 +43,7 @@ val_dataset = generate_qa_embedding_pairs(
 )
 ```
 
-Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model:
 
 ```python
 from llama_index.finetuning import SentenceTransformersFinetuneEngine
@@ -57,7 +57,7 @@ finetune_engine = SentenceTransformersFinetuneEngine(
 finetune_engine.finetune()
 embed_model = finetune_engine.get_finetuned_model()
 ```
-This saves the fine tuned embedding model in `tuned_model` folder. This al
+This saves the fine tuned embedding model in `tuned_model` folder.
 
 # Evaluation results
 In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 

docs/src/hybrid_search/eval.md

@@ -3,22 +3,22 @@
 Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
 
 ## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step: reranking.
 There are two approaches for reranking search results from multiple sources.
 
-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example: Weighted linear combination of semantic search & keyword-based search results.
 
-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models
 
-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset or application specific so it's hard to generalize.
 
 ### Example evaluation of hybrid search with Reranking
 
-Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+Here's some evaluation numbers from an experiment comparing these rerankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
 
 <b> With OpenAI ada2 embedding </b>
 
-Vector Search baseline - `0.64`
+Vector Search baseline: `0.64`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
@@ -33,7 +33,7 @@ Vector Search baseline - `0.64`
 
 <b> With OpenAI embedding-v3-small </b>
 
-Vector Search baseline - `0.59`
+Vector Search baseline: `0.59`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |

docs/src/hybrid_search/hybrid_search.md

@@ -5,57 +5,46 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
-```python
-import os
+=== "Sync API"
 
-import lancedb
-import openai
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:import-os"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
+    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search"
+    ```
+=== "Async API"
 
-db = lancedb.connect("~/.lancedb")
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:import-os"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
+    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
+    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
+    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
+    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search_async"
+    ```
 
-# Ingest embedding function in LanceDB table
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-    openai.api_key = "sk-..."
-embeddings = get_registry().get("openai").create()
-
-class Documents(LanceModel):
-    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
-    text: str = embeddings.SourceField()
-
-table = db.create_table("documents", schema=Documents)
-
-data = [
-    { "text": "rebel spaceships striking from a hidden base"},
-    { "text": "have won their first victory against the evil Galactic Empire"},
-    { "text": "during the battle rebel spies managed to steal secret plans"},
-    { "text": "to the Empire's ultimate weapon the Death Star"}
-]
-
-# ingest docs with auto-vectorization
-table.add(data)
-
-# Create a fts index before the hybrid search
-table.create_fts_index("text")
-# hybrid search with default re-ranker
-results = table.search("flower moon", query_type="hybrid").to_pandas()
-```
 !!! Note
     You can also pass the vector and text query manually. This is useful if you're not using the embedding API or if you're using a separate embedder service.
 ### Explicitly passing the vector and text query
-```python
-vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
-text_query = "flower moon"
-results = table.search(query_type="hybrid")
-                .vector(vector_query)
-                .text(text_query)
-                .limit(5)
-                .to_pandas()
+=== "Sync API"
 
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text_async"
+    ```
 
 By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
 
@@ -68,7 +57,7 @@ By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion scor
 
 
 ## Available Rerankers
-LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. 
+LanceDB provides a number of rerankers out of the box. You can use any of these rerankers by passing them to the `rerank()` method. 
 Go to [Rerankers](../reranking/index.md) to learn more about using the available rerankers and implementing custom rerankers.
 
 

docs/src/migration.md

@@ -1,81 +1,14 @@
 # Rust-backed Client Migration Guide
 
-In an effort to ensure all clients have the same set of capabilities we have begun migrating the
-python and node clients onto a common Rust base library. In python, this new client is part of
-the same lancedb package, exposed as an asynchronous client. Once the asynchronous client has
-reached full functionality we will begin migrating the synchronous library to be a thin wrapper
-around the asynchronous client.
+In an effort to ensure all clients have the same set of capabilities we have
+migrated the Python and Node clients onto a common Rust base library. In Python,
+both the synchronous and asynchronous clients are based on this implementation.
+In Node, the new client is available as `@lancedb/lancedb`, which replaces
+the existing `vectordb` package.
 
-This guide describes the differences between the two APIs and will hopefully assist users
+This guide describes the differences between the two Node APIs and will hopefully assist users
 that would like to migrate to the new API.
 
-## Python
-### Closeable Connections
-
-The Connection now has a `close` method. You can call this when
-you are done with the connection to eagerly free resources. Currently
-this is limited to freeing/closing the HTTP connection for remote
-connections. In the future we may add caching or other resources to
-native connections so this is probably a good practice even if you
-aren't using remote connections.
-
-In addition, the connection can be used as a context manager which may
-be a more convenient way to ensure the connection is closed.
-
-```python
-import lancedb
-
-async def my_async_fn():
-    with await lancedb.connect_async("my_uri") as db:
-        print(await db.table_names())
-```
-
-It is not mandatory to call the `close` method. If you do not call it
-then the connection will be closed when the object is garbage collected.
-
-### Closeable Table
-
-The Table now also has a `close` method, similar to the connection. This
-can be used to eagerly free the cache used by a Table object. Similar to
-the connection, it can be used as a context manager and it is not mandatory
-to call the `close` method.
-
-#### Changes to Table APIs
-
-- Previously `Table.schema` was a property. Now it is an async method.
-- The method `Table.__len__` was removed and `len(table)` will no longer
-  work. Use `Table.count_rows` instead.
-
-#### Creating Indices
-
-The `Table.create_index` method is now used for creating both vector indices
-and scalar indices. It currently requires a column name to be specified (the
-column to index). Vector index defaults are now smarter and scale better with
-the size of the data.
-
-To specify index configuration details you will need to specify which kind of
-index you are using.
-
-#### Querying
-
-The `Table.search` method has been renamed to `AsyncTable.vector_search` for
-clarity.
-
-### Features not yet supported
-
-The following features are not yet supported by the asynchronous API. However,
-we plan to support them soon.
-
-- You cannot specify an embedding function when creating or opening a table.
-  You must calculate embeddings yourself if using the asynchronous API
-- The merge insert operation is not supported in the asynchronous API
-- Cleanup / compact / optimize indices are not supported in the asynchronous API
-- add / alter columns is not supported in the asynchronous API
-- The asynchronous API does not yet support any full text search or reranking
-  search
-- Remote connections to LanceDb Cloud are not yet supported.
-- The method Table.head is not yet supported.
-
 ## TypeScript/JavaScript
 
 For JS/TS users, we offer a brand new SDK [@lancedb/lancedb](https://www.npmjs.com/package/@lancedb/lancedb)
@@ -133,7 +66,7 @@ the size of the data.
 
 ### Embedding Functions
 
-The embedding API has been completely reworked, and it now more closely resembles the Python API, including the new [embedding registry](./js/classes/embedding.EmbeddingFunctionRegistry.md)
+The embedding API has been completely reworked, and it now more closely resembles the Python API, including the new [embedding registry](./js/classes/embedding.EmbeddingFunctionRegistry.md):
 
 === "vectordb (deprecated)"
 

docs/src/notebooks/embedding_tuner.ipynb

@@ -207,7 +207,7 @@
       "cell_type": "markdown",
       "source": [
         "## The dataset\n",
-        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retreiver.\n",
+        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retriever.\n",
         "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo\n"
       ],
       "metadata": {

docs/src/notebooks/hybrid_search.ipynb

@@ -477,7 +477,7 @@
    "source": [
     "## Vector Search\n",
     "\n",
-    "avg latency - `3.48 ms ± 71.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
+    "Average latency: `3.48 ms ± 71.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
    ]
   },
   {
@@ -597,7 +597,7 @@
     "`LinearCombinationReranker(weight=0.7)` is used as the default reranker for reranking the hybrid search results if the reranker isn't specified explicitly.\n",
     "The `weight` param controls the weightage provided to vector search score. The weight of `1-weight` is applied to FTS scores when reranking.\n",
     "\n",
-    "Latency - `71 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
+    "Latency: `71 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
    ]
   },
   {
@@ -675,9 +675,9 @@
    },
    "source": [
     "### Cohere Reranker\n",
-    "This uses Cohere's Reranking API to re-rank  the results. It accepts the reranking model name as a parameter. By Default it uses the english-v3 model but you can easily switch to a multi-lingual model.\n",
+    "This uses Cohere's Reranking API to re-rank  the results. It accepts the reranking model name as a parameter. By default it uses the english-v3 model but you can easily switch to a multi-lingual model.\n",
     "\n",
-    "latency - `605 ms ± 78.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
+    "Latency: `605 ms ± 78.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
    ]
   },
   {
@@ -1165,7 +1165,7 @@
    },
    "source": [
     "### ColBERT Reranker\n",
-    "Colber Reranker is powered by ColBERT model. It runs locally using the huggingface implementation.\n",
+    "Colbert Reranker is powered by ColBERT model. It runs locally using the huggingface implementation.\n",
     "\n",
     "Latency - `950 ms ± 5.78 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`\n",
     "\n",
@@ -1489,9 +1489,9 @@
    },
    "source": [
     "### Cross Encoder Reranker\n",
-    "Uses cross encoder models are rerankers. Uses sentence transformer implemntation locally\n",
+    "Uses cross encoder models are rerankers. Uses sentence transformer implementation locally\n",
     "\n",
-    "Latency - `1.38 s ± 64.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
+    "Latency: `1.38 s ± 64.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
    ]
   },
   {
@@ -1771,10 +1771,10 @@
    "source": [
     "### (Experimental) OpenAI Reranker\n",
     "\n",
-    "This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental. You might run out of token limit so set the search limits based on your token limit.\n",
-    "NOTE: It is recommended to use `gpt-4-turbo-preview`, older models might lead to bad behaviour\n",
+    "This prompts a chat model to rerank results and is not a dedicated reranker model. This should be treated as experimental. You might exceed the token limit so set the search limits based on your token limit.\n",
+    "NOTE: It is recommended to use `gpt-4-turbo-preview` as older models might lead to bad behaviour\n",
     "\n",
-    "Latency - `Can take 10s of seconds if using GPT-4 model`"
+    "Latency: `Can take 10s of seconds if using GPT-4 model`"
    ]
   },
   {
@@ -1817,7 +1817,7 @@
    },
    "source": [
     "## Use your custom Reranker\n",
-    "Hybrid search in LanceDB is designed to be very flexible. You can easily plug in your own Re-reranking logic. To do so, you simply need to implement the base Reranker class"
+    "Hybrid search in LanceDB is designed to be very flexible. You can easily plug in your own Re-reranking logic. To do so, you simply need to implement the base Reranker class:"
    ]
   },
   {
@@ -1849,9 +1849,9 @@
    "source": [
     "### Custom Reranker based on CohereReranker\n",
     "\n",
-    "For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.\n",
+    "For the sake of simplicity let's build a custom reranker that enhances the Cohere Reranker by accepting a filter query, and accepts other CohereReranker params as kwargs.\n",
     "\n",
-    "For this toy example let's say we want to get rid of docs that represent a table of contents, appendix etc. as these are semantically close of representing costs but this isn't something we are interested in because they don't represent the specific reasons why operating costs were high. They simply represent the costs."
+    "For this toy example let's say we want to get rid of docs that represent a table of contents or appendix, as these are semantically close to representing costs but don't represent the specific reasons why operating costs were high."
    ]
   },
   {
@@ -1969,7 +1969,7 @@
     "id": "b3b5464a-7252-4eab-aaac-9b0eae37496f"
    },
    "source": [
-    "As you can see the document containing the Table of contetnts of spending no longer shows up"
+    "As you can see, the document containing the table of contents no longer shows up."
    ]
   }
  ],

docs/src/notebooks/lancedb_reranking.ipynb

@@ -49,7 +49,7 @@
       },
       "source": [
         "## What is a retriever\n",
-        "VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.\n",
+        "VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.\n",
         "\n",
         "<img src=\"https://llmstack.ai/assets/images/rag-f517f1f834bdbb94a87765e0edd40ff2.png\" />\n",
         "\n",
@@ -64,7 +64,7 @@
         "- Fine-tuning the embedding models\n",
         "- Using different embedding models\n",
         "\n",
-        "Obviously, the above list is not exhaustive. There are other subtler ways that can improve retrieval performance like experimenting chunking algorithms, using different distance/similarity metrics etc. But for brevity, we'll only cover high level and more impactful techniques here.\n",
+        "Obviously, the above list is not exhaustive. There are other subtler ways that can improve retrieval performance like alternative chunking algorithms, using different distance/similarity metrics, and more. For brevity, we'll only cover high level and more impactful techniques here.\n",
         "\n"
       ]
     },
@@ -77,7 +77,7 @@
         "# LanceDB\n",
         "- Multimodal DB for AI\n",
         "- Powered by an innovative & open-source in-house file format\n",
-        "- 0 Setup\n",
+        "- Zero setup\n",
         "- Scales up on disk storage\n",
         "- Native support for vector, full-text(BM25) and hybrid search\n",
         "\n",
@@ -92,8 +92,8 @@
       },
       "source": [
         "## The dataset\n",
-        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retreiver.\n",
-        "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo\n"
+        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retriever.\n",
+        "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo.\n"
       ]
     },
     {
@@ -594,10 +594,10 @@
       },
       "source": [
         "## Ingestion\n",
-        "Let us now ingest the contexts in LanceDB\n",
+        "Let us now ingest the contexts in LanceDB. The steps will be:\n",
         "\n",
         "- Create a schema (Pydantic or Pyarrow)\n",
-        "- Select an embedding model from LanceDB Embedding API (Allows automatic vectorization of data)\n",
+        "- Select an embedding model from LanceDB Embedding API (to allow automatic vectorization of data)\n",
         "- Ingest the contexts\n"
       ]
     },
@@ -841,7 +841,7 @@
       },
       "source": [
         "## Different Query types in LanceDB\n",
-        "LanceDB allows switching query types with by setting `query_type` argument, which defaults to `vector` when using Embedding API. In this example we'll use `JinaReranker` which is one of many rerankers supported by LanceDB\n",
+        "LanceDB allows switching query types with by setting `query_type` argument, which defaults to `vector` when using Embedding API. In this example we'll use `JinaReranker` which is one of many rerankers supported by LanceDB.\n",
         "\n",
         "### Vector search:\n",
         "Vector search\n",
@@ -1446,11 +1446,11 @@
       "source": [
         "## Takeaways & Tradeoffs\n",
         "\n",
-        "* **Easiest method to significantly improve accuracy** Using Hybrid search and/or rerankers can significantly improve retrieval performance without spending any additional time or effort on tuning embedding models, generators, or dissecting the dataset.\n",
+        "* **Rerankers significantly improve accuracy at little cost.** Using Hybrid search and/or rerankers can significantly improve retrieval performance without spending any additional time or effort on tuning embedding models, generators, or dissecting the dataset.\n",
         "\n",
         "* **Reranking is an expensive operation.** Depending on the type of reranker you choose, they can incur significant latecy to query times. Although some API-based rerankers can be significantly faster.\n",
         "\n",
-        "*  When using models locally, having a warmed-up GPU environment will significantly reduce latency. This is specially useful if the application doesn't need to be strcitly realtime. The tradeoff being GPU resources."
+        "* **Pre-warmed GPU environments reduce latency.**  When using models locally, having a warmed-up GPU environment will significantly reduce latency. This is especially useful if the application doesn't need to be strictly realtime. Pre-warming comes at the expense of GPU resources."
       ]
     },
     {
@@ -1504,4 +1504,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
+}

docs/src/python/pandas_and_pyarrow.md

@@ -8,54 +8,55 @@ and PyArrow. The sequence of steps in a typical workflow is shown below.
 
 First, we need to connect to a LanceDB database.
 
-```py
+=== "Sync API"
 
-import lancedb
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb"
+    ```
+=== "Async API"
 
-db = lancedb.connect("data/sample-lancedb")
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-lancedb"
+    --8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb_async"
+    ```
 
 We can load a Pandas `DataFrame` to LanceDB directly.
 
-```py
-import pandas as pd
+=== "Sync API"
 
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-pandas"
+    --8<-- "python/python/tests/docs/test_python.py:create_table_pandas"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-pandas"
+    --8<-- "python/python/tests/docs/test_python.py:create_table_pandas_async"
+    ```
 
 Similar to the [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html) method, LanceDB's
 [`db.create_table()`](python.md/#lancedb.db.DBConnection.create_table) accepts data in a variety of forms.
 
 If you have a dataset that is larger than memory, you can create a table with `Iterator[pyarrow.RecordBatch]` to lazily load the data:
 
-```py
+=== "Sync API"
 
-from typing import Iterable
-import pyarrow as pa
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-iterable"
+    --8<-- "python/python/tests/docs/test_python.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_python.py:make_batches"
+    --8<-- "python/python/tests/docs/test_python.py:create_table_iterable"
+    ```
+=== "Async API"
 
-def make_batches() -> Iterable[pa.RecordBatch]:
-    for i in range(5):
-        yield pa.RecordBatch.from_arrays(
-            [
-                pa.array([[3.1, 4.1], [5.9, 26.5]]),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"])
-
-schema=pa.schema([
-    pa.field("vector", pa.list_(pa.float32())),
-    pa.field("item", pa.utf8()),
-    pa.field("price", pa.float32()),
-])
-
-table = db.create_table("iterable_table", data=make_batches(), schema=schema)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:import-iterable"
+    --8<-- "python/python/tests/docs/test_python.py:import-pyarrow"
+    --8<-- "python/python/tests/docs/test_python.py:make_batches"
+    --8<-- "python/python/tests/docs/test_python.py:create_table_iterable_async"
+    ```
 
 You will find detailed instructions of creating a LanceDB dataset in
 [Getting Started](../basic.md#quick-start) and [API](python.md/#lancedb.db.DBConnection.create_table)
@@ -65,15 +66,16 @@ sections.
 
 We can now perform similarity search via the LanceDB Python API.
 
-```py
-# Open the table previously created.
-table = db.open_table("pd_table")
+=== "Sync API"
 
-query_vector = [100, 100]
-# Pandas DataFrame
-df = table.search(query_vector).limit(1).to_pandas()
-print(df)
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:vector_search"
+    ```
+=== "Async API"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:vector_search_async"
+    ```
 
 ```
     vector     item  price    _distance
@@ -83,16 +85,13 @@ print(df)
 If you have a simple filter, it's faster to provide a `where` clause to LanceDB's `search` method.
 For more complex filters or aggregations, you can always resort to using the underlying `DataFrame` methods after performing a search.
 
-```python
+=== "Sync API"
 
-# Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_pandas()
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:vector_search_with_filter"
+    ```
+=== "Async API"
 
-# Apply the filter via Pandas
-df = results = table.search([100, 100]).to_pandas()
-results = df[df.price < 15]
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-```
+    ```python
+    --8<-- "python/python/tests/docs/test_python.py:vector_search_with_filter_async"
+    ```

docs/src/python/polars_arrow.md

@@ -2,38 +2,29 @@
 
 LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow under the hood. A deeper integration between Lance Tables and Polars DataFrames is in progress, but at the moment, you can read a Polars DataFrame into LanceDB and output the search results from a query to a Polars DataFrame.
 
+
 ## Create & Query LanceDB Table
 
 ### From Polars DataFrame
 
 First, we connect to a LanceDB database.
 
-```py
-import lancedb
 
-db = lancedb.connect("data/polars-lancedb")
+```py
+--8<-- "python/python/tests/docs/test_python.py:import-lancedb"
+--8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb"
 ```
 
 We can load a Polars `DataFrame` to LanceDB directly.
 
 ```py
-import polars as pl
-
-data = pl.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pl_table", data=data)
+--8<-- "python/python/tests/docs/test_python.py:import-polars"
+--8<-- "python/python/tests/docs/test_python.py:create_table_polars"
 ```
-
 We can now perform similarity search via the LanceDB Python API.
 
 ```py
-query = [3.0, 4.0]
-result = table.search(query).limit(1).to_polars()
-print(result)
-print(type(result))
+--8<-- "python/python/tests/docs/test_python.py:vector_search_polars"
 ```
 
 In addition to the selected columns, LanceDB also returns a vector
@@ -59,33 +50,16 @@ Note that the type of the result from a table search is a Polars DataFrame.
 Alternately, we can create an empty LanceDB Table using a Pydantic schema and populate it with a Polars DataFrame.
 
 ```py
-import polars as pl
-from lancedb.pydantic import Vector, LanceModel
-
-
-class Item(LanceModel):
-    vector: Vector(2)
-    item: str
-    price: float
-
-data = {
-    "vector": [[3.1, 4.1]],
-    "item": "foo",
-    "price": 10.0,
-}
-
-table = db.create_table("test_table", schema=Item)
-df = pl.DataFrame(data)
-# Add Polars DataFrame to table
-table.add(df)
+--8<-- "python/python/tests/docs/test_python.py:import-polars"
+--8<-- "python/python/tests/docs/test_python.py:import-lancedb-pydantic"
+--8<-- "python/python/tests/docs/test_python.py:class_Item"
+--8<-- "python/python/tests/docs/test_python.py:create_table_pydantic"
 ```
 
 The table can now be queried as usual.
 
 ```py
-result = table.search([3.0, 4.0]).limit(1).to_polars()
-print(result)
-print(type(result))
+--8<-- "python/python/tests/docs/test_python.py:vector_search_polars"
 ```
 
 ```
@@ -108,8 +82,7 @@ As you iterate on your application, you'll likely need to work with the whole ta
 LanceDB tables can also be converted directly into a polars LazyFrame for further processing.
 
 ```python
-ldf = table.to_polars()
-print(type(ldf))
+--8<-- "python/python/tests/docs/test_python.py:dump_table_lazyform"
 ```
 
 Unlike the search result from a query, we can see that the type of the result is a LazyFrame.
@@ -121,7 +94,7 @@ Unlike the search result from a query, we can see that the type of the result is
 We can now work with the LazyFrame as we would in Polars, and collect the first result.
 
 ```python
-print(ldf.first().collect())
+--8<-- "python/python/tests/docs/test_python.py:print_table_lazyform"
 ```
 
 ```

docs/src/python/python.md

@@ -47,6 +47,8 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
 
+::: lancedb.embeddings.base.EmbeddingFunctionConfig
+
 ::: lancedb.embeddings.base.EmbeddingFunction
 
 ::: lancedb.embeddings.base.TextEmbeddingFunction
@@ -127,8 +129,16 @@ lists the indices that LanceDb supports.
 
 ::: lancedb.index.LabelList
 
+::: lancedb.index.FTS
+
 ::: lancedb.index.IvfPq
 
+::: lancedb.index.HnswPq
+
+::: lancedb.index.HnswSq
+
+::: lancedb.index.IvfFlat
+
 ## Querying (Asynchronous)
 
 Queries allow you to return data from your database. Basic queries can be

docs/src/python/saas-python.md

@@ -17,4 +17,8 @@ pip install lancedb
 ## Table
 
 ::: lancedb.remote.table.RemoteTable
-
+    options:
+        filters:
+            - "!cleanup_old_versions"
+            - "!compact_files"
+            - "!optimize"

docs/src/rag/adaptive_rag.md

@@ -2,7 +2,7 @@
 ====================================================================
 Adaptive RAG introduces a RAG technique that combines query analysis with self-corrective RAG. 
 
-For Query Analysis, it uses a small classifier(LLM), to decide the query’s complexity. Query Analysis helps routing smoothly to adjust between different retrieval strategies No retrieval, Single-shot RAG or Iterative RAG.
+For Query Analysis, it uses a small classifier(LLM), to decide the query’s complexity. Query Analysis guides adjustment between different retrieval strategies: No retrieval, Single-shot RAG or Iterative RAG.
 
 **[Official Paper](https://arxiv.org/pdf/2403.14403)**
 
@@ -12,9 +12,9 @@ For Query Analysis, it uses a small classifier(LLM), to decide the query’s com
   </figcaption>
 </figure>
 
-**[Offical Implementation](https://github.com/starsuzi/Adaptive-RAG)**
+**[Official Implementation](https://github.com/starsuzi/Adaptive-RAG)**
 
-Here’s a code snippet for query analysis
+Here’s a code snippet for query analysis:
 
 ```python
 from langchain_core.prompts import ChatPromptTemplate
@@ -35,7 +35,7 @@ llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
 structured_llm_router = llm.with_structured_output(RouteQuery)
 ```
 
-For defining and querying retriever
+The following example defines and queries a retriever:
 
 ```python
 # add documents in LanceDB
@@ -48,4 +48,4 @@ retriever = vectorstore.as_retriever()
 # query using defined retriever
 question = "How adaptive RAG works"
 docs = retriever.get_relevant_documents(question)
-```
+```

docs/src/rag/advanced_techniques/flare.md

@@ -11,7 +11,7 @@ FLARE, stands for Forward-Looking Active REtrieval augmented generation is a gen
 
 [![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)
 
-Here’s a code snippet for using FLARE with Langchain
+Here’s a code snippet for using FLARE with Langchain:
 
 ```python
 from langchain.vectorstores import LanceDB
@@ -35,4 +35,4 @@ flare = FlareChain.from_llm(llm=llm,retriever=vector_store_retriever,max_generat
 result = flare.run(input_text)
 ```
 
-[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)

docs/src/rag/advanced_techniques/hyde.md

@@ -11,7 +11,7 @@ HyDE, stands for Hypothetical Document Embeddings is an approach used for precis
 
 [![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb)
 
-Here’s a code snippet for using HyDE with Langchain
+Here’s a code snippet for using HyDE with Langchain:
 
 ```python
 from langchain.llms import OpenAI

docs/src/rag/agentic_rag.md

@@ -1,6 +1,6 @@
 **Agentic RAG 🤖**
 ====================================================================
-Agentic RAG is Agent-based RAG introduces an advanced framework for answering questions by using intelligent agents instead of just relying on large language models. These agents act like expert researchers, handling complex tasks such as detailed planning, multi-step reasoning, and using external tools. They navigate multiple documents, compare information, and generate accurate answers. This system is easily scalable, with each new document set managed by a sub-agent, making it a powerful tool for tackling a wide range of information needs.
+Agentic RAG introduces an advanced framework for answering questions by using intelligent agents instead of just relying on large language models. These agents act like expert researchers, handling complex tasks such as detailed planning, multi-step reasoning, and using external tools. They navigate multiple documents, compare information, and generate accurate answers. This system is easily scalable, with each new document set managed by a sub-agent, making it a powerful tool for tackling a wide range of information needs.
 
 <figure markdown="span">
   ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/agentic_rag.png)
@@ -9,7 +9,7 @@ Agentic RAG is Agent-based RAG introduces an advanced framework for answering qu
 
 [![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)
 
-Here’s a code snippet for defining retriever using Langchain
+Here’s a code snippet for defining retriever using Langchain:
 
 ```python
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -41,7 +41,7 @@ retriever = vectorstore.as_retriever()
 
 ```
 
-Agent that formulates an improved query for better retrieval results and then grades the retrieved documents
+Here is an agent that formulates an improved query for better retrieval results and then grades the retrieved documents:
 
 ```python
 def grade_documents(state) -> Literal["generate", "rewrite"]:
@@ -98,4 +98,4 @@ def rewrite(state):
     return {"messages": [response]}
 ```
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)

docs/src/rag/corrective_rag.md

@@ -4,7 +4,7 @@
 Corrective-RAG (CRAG) is a strategy for Retrieval-Augmented Generation (RAG) that includes self-reflection and self-grading of retrieved documents. Here’s a simplified breakdown of the steps involved:
 
 1. **Relevance Check**: If at least one document meets the relevance threshold, the process moves forward to the generation phase.
-2. **Knowledge Refinement**: Before generating an answer, the process refines the knowledge by dividing the document into smaller segments called "knowledge strips."
+2. **Knowledge Refinement**: Before generating an answer, the process refines the knowledge by dividing the document into smaller segments called "knowledge strips".
 3. **Grading and Filtering**: Each "knowledge strip" is graded, and irrelevant ones are filtered out.
 4. **Additional Data Source**: If all documents are below the relevance threshold, or if the system is unsure about their relevance, it will seek additional information by performing a web search to supplement the retrieved data.
 
@@ -19,11 +19,11 @@ Above steps are mentioned in
 
 Corrective Retrieval-Augmented Generation (CRAG) is a method that works like a **built-in fact-checker**.
 
-**[Offical Implementation](https://github.com/HuskyInSalt/CRAG)**
+**[Official Implementation](https://github.com/HuskyInSalt/CRAG)**
 
 [![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)
 
-Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), and retrieves the relevant documents.
+Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), and retrieves the relevant documents:
 
 ```python
 import pandas as pd
@@ -115,6 +115,6 @@ def grade_documents(state):
     }
 ```
 
-Check Colab for the Implementation of CRAG with Langgraph 
+Check Colab for the Implementation of CRAG with Langgraph:
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)

docs/src/rag/graph_rag.md

@@ -6,7 +6,7 @@ One of the main benefits of Graph RAG is its ability to capture and represent co
 
 **[Official Paper](https://arxiv.org/pdf/2404.16130)**
 
-**[Offical Implementation](https://github.com/microsoft/graphrag)**
+**[Official Implementation](https://github.com/microsoft/graphrag)**
 
 [Microsoft Research Blog](https://www.microsoft.com/en-us/research/blog/graphrag-unlocking-llm-discovery-on-narrative-private-data/)
 
@@ -39,16 +39,16 @@ python3 -m graphrag.index --root dataset-dir
 
 - **Execute Query**
 
-Global Query Execution gives a broad overview of dataset
+Global Query Execution gives a broad overview of dataset:
 
 ```bash
 python3 -m graphrag.query --root dataset-dir --method global "query-question"
 ```
 
-Local Query Execution gives a detailed and specific answers based on the context of the entities
+Local Query Execution gives a detailed and specific answers based on the context of the entities:
 
 ```bash
 python3 -m graphrag.query --root  dataset-dir --method local "query-question"
 ```
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Graphrag/main.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Graphrag/main.ipynb)

docs/src/rag/multi_head_rag.md

@@ -15,7 +15,7 @@ MRAG is cost-effective and energy-efficient because it avoids extra LLM queries,
 
 **[Official Implementation](https://github.com/spcl/MRAG)**
 
-Here’s a code snippet for defining different embedding spaces with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/)
+Here’s a code snippet for defining different embedding spaces with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/):
 
 ```python
 import lancedb
@@ -44,6 +44,6 @@ class Space3(LanceModel):
     vector: Vector(model3.ndims()) = model3.VectorField()
 ```
 
-Create different tables using defined embedding spaces, then make queries to each embedding space. Use the resulted closest documents from each embedding space to generate answers.
+Create different tables using defined embedding spaces, then make queries to each embedding space. Use the resulting closest documents from each embedding space to generate answers.
 
 

docs/src/rag/self_rag.md

@@ -1,6 +1,6 @@
 **Self RAG 🤳**
 ====================================================================
-Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better retrieved information, generated text, and  checking their own work, all without losing their flexibility. Unlike the traditional Retrieval-Augmented Generation (RAG) method, Self-RAG retrieves information as needed, can skip retrieval if not needed, and evaluates its own output while generating text. It also uses a process to pick the best output based on different preferences.
+Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better retrieved information, generated text, and validation, without loss of flexibility. Unlike the traditional Retrieval-Augmented Generation (RAG) method, Self-RAG retrieves information as needed, can skip retrieval if not needed, and evaluates its own output while generating text. It also uses a process to pick the best output based on different preferences.
 
 **[Official Paper](https://arxiv.org/pdf/2310.11511)**
 
@@ -10,11 +10,11 @@ Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better re
   </figcaption>
 </figure>
 
-**[Offical Implementation](https://github.com/AkariAsai/self-rag)**
+**[Official Implementation](https://github.com/AkariAsai/self-rag)**
 
 Self-RAG starts by generating a response without retrieving extra info if it's not needed. For questions that need more details, it retrieves to get the necessary information.
 
-Here’s a code snippet for defining retriever using Langchain
+Here’s a code snippet for defining retriever using Langchain:
 
 ```python
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -46,7 +46,7 @@ retriever = vectorstore.as_retriever()
 
 ```
 
-Functions that grades the retrieved documents and if required formulates an improved query for better retrieval results  
+The following functions grade the retrieved documents and formulate an improved query for better retrieval results, if required:
 
 ```python
 def grade_documents(state) -> Literal["generate", "rewrite"]:
@@ -93,4 +93,4 @@ def rewrite(state):
     model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
     response = model.invoke(msg)
     return {"messages": [response]}
-```
+```

docs/src/rag/sfr_rag.md

@@ -1,8 +1,8 @@
 **SFR RAG 📑**
 ====================================================================
-Salesforce AI Research introduces SFR-RAG, a 9-billion-parameter language model trained with a significant emphasis on reliable, precise, and faithful contextual generation abilities specific to real-world RAG use cases and relevant agentic tasks. They include precise factual knowledge extraction, distinguishing relevant against distracting contexts, citing appropriate sources along with answers, producing complex and multi-hop reasoning over multiple contexts, consistent format following, as well as refraining from hallucination over unanswerable queries.
+Salesforce AI Research introduced SFR-RAG, a 9-billion-parameter language model trained with a significant emphasis on reliable, precise, and faithful contextual generation abilities specific to real-world RAG use cases and relevant agentic tasks. It targets precise factual knowledge extraction, distinction between relevant and distracting contexts, citation of appropriate sources along with answers, production of complex and multi-hop reasoning over multiple contexts, consistent format following, as well as minimization of hallucination over unanswerable queries.
 
-**[Offical Implementation](https://github.com/SalesforceAIResearch/SFR-RAG)**
+**[Official Implementation](https://github.com/SalesforceAIResearch/SFR-RAG)**
 
 <figure markdown="span">
   ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/salesforce_contextbench.png)

docs/src/reranking/answerdotai.md

@@ -1,7 +1,6 @@
 # AnswersDotAI Rerankers
 
-This integration allows using answersdotai's rerankers to rerank the search results. [Rerankers](https://github.com/AnswerDotAI/rerankers)
-A lightweight, low-dependency, unified API to use all common reranking and cross-encoder models.
+This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/rerankers) to rerank the search results, providing a lightweight, low-dependency, unified API to use all common reranking and cross-encoder models.
 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
@@ -45,10 +44,10 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here - https://github.com/AnswerDotAI/rerankers |
+| `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here: https://github.com/AnswerDotAI/rerankers. |
 | `model_name` | `str` | `"answerdotai/answerai-colbert-small-v1"` | The name of the reranker model to use. |
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
 
 
 
@@ -58,17 +57,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/cohere.md

@@ -1,6 +1,6 @@
 # Cohere Reranker
 
-This re-ranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this reranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
 
 
 !!! note
@@ -62,17 +62,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/colbert.md

@@ -1,6 +1,6 @@
 # ColBERT Reranker
 
-This re-ranker uses ColBERT model to rerank the search results. You can use this re-ranker by passing `ColbertReranker()` to the `rerank()` method. 
+This reranker uses ColBERT model to rerank the search results. You can use this reranker by passing `ColbertReranker()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
@@ -46,7 +46,7 @@ Accepted Arguments
 | `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
 
 
 ## Supported Scores for each query type
@@ -55,17 +55,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/cross_encoder.md

@@ -1,6 +1,6 @@
 # Cross Encoder Reranker
 
-This re-ranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this re-ranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
+This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
@@ -46,7 +46,7 @@ Accepted Arguments
 | `model_name` | `str` | `""cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
 
 ## Supported Scores for each query type
 You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
@@ -54,17 +54,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/custom_reranker.md

@@ -1,9 +1,10 @@
 ## Building Custom Rerankers
 You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Optionally, you can also implement the `rerank_vector()` and `rerank_fts()` methods if you want to support reranking for vector and FTS search separately.
-Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
 
 The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
 
+Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores:
+
 ```python
 
 from lancedb.rerankers import Reranker
@@ -42,7 +43,7 @@ class MyReranker(Reranker):
 ```
 
 ### Example of a Custom Reranker
-For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
+For the sake of simplicity let's build custom reranker that enhances the Cohere Reranker by accepting a filter query, and accepts other CohereReranker params as kwargs.
 
 ```python
 
@@ -83,6 +84,6 @@ class ModifiedCohereReranker(CohereReranker):
 ```
 
 !!! tip
-    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be convered to other data types like pandas dataframe, pydict, pylist etc.
+    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be converted to other data types like pandas dataframe, pydict, pylist etc.
 
-    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.
+    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/reranking/index.md

@@ -13,7 +13,7 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 
 
 ## Using a Reranker
-Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
+Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder:
 
 ```python
 import lancedb
@@ -36,14 +36,14 @@ tbl = db.create_table("test", data)
 reranker = CohereReranker(api_key="your_api_key")
 
 # Run vector search with a reranker
-result = tbl.query("hello").rerank(reranker).to_list() 
+result = tbl.search("hello").rerank(reranker).to_list() 
 
 # Run FTS search with a reranker
-result = tbl.query("hello", query_type="fts").rerank(reranker).to_list()
+result = tbl.search("hello", query_type="fts").rerank(reranker).to_list()
 
 # Run hybrid search with a reranker
 tbl.create_fts_index("text")
-result = tbl.query("hello", query_type="hybrid").rerank(reranker).to_list()
+result = tbl.search("hello", query_type="hybrid").rerank(reranker).to_list()
 ```
 
 ### Multi-vector reranking
@@ -64,7 +64,7 @@ reranked = reranker.rerank_multivector([res1, res2, res3],  deduplicate=True)
 ```
     
 ## Available Rerankers
-LanceDB comes with some built-in rerankers. Here are some of the rerankers that are available in LanceDB:
+LanceDB comes with the following built-in rerankers:
 
 - [Cohere Reranker](./cohere.md)
 - [Cross Encoder Reranker](./cross_encoder.md)
@@ -78,4 +78,4 @@ LanceDB comes with some built-in rerankers. Here are some of the rerankers that
 
 ## Creating Custom Rerankers
 
-LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.
+LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.

docs/src/reranking/jina.md

@@ -1,6 +1,6 @@
 # Jina Reranker
 
-This re-ranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this re-ranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this reranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
 
 
 !!! note
@@ -48,11 +48,11 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker/|
+| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker. |
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
 | `api_key` | `str` | `None` | The API key for the Jina API. If not provided, the `JINA_API_KEY` environment variable is used. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
 
 
 
@@ -62,17 +62,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/linear_combination.md

@@ -1,9 +1,9 @@
 # Linear Combination Reranker
 
 !!! note
-    This is depricated. It is recommended to use the `RRFReranker` instead, if you want to use a score based reranker.
+    This is deprecated. It is recommended to use the `RRFReranker` instead, if you want to use a score-based reranker.
 
-It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
 
 !!! note
     Supported Query Types: Hybrid
@@ -51,5 +51,5 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
+| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |

docs/src/reranking/openai.md

@@ -1,11 +1,11 @@
 # OpenAI Reranker (Experimental)
 
-This re-ranker uses OpenAI chat model to rerank the search results. You can use this re-ranker by passing `OpenAI()` to the `rerank()` method. 
+This reranker uses OpenAI chat model to rerank the search results. You can use this reranker by passing `OpenAI()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
 !!! warning
-    This re-ranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
+    This reranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
 
 ```python
 import numpy
@@ -47,7 +47,7 @@ Accepted Arguments
 | --- | --- | --- | --- |
 | `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
 | `api_key` | str | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable.
 
 
@@ -57,17 +57,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/rrf.md

@@ -1,6 +1,6 @@
 # Reciprocal Rank Fusion Reranker
 
-This is the default re-ranker used by LanceDB hybrid search. Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
+This is the default reranker used by LanceDB hybrid search. Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
 
 
 !!! note
@@ -39,7 +39,7 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical |
+| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. |
 | `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. |
 
 
@@ -49,5 +49,5 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. |
+| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |

docs/src/reranking/voyageai.md

@@ -2,7 +2,7 @@
 
 Voyage AI provides cutting-edge embedding and rerankers.
 
-This re-ranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this re-ranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
 
 
 !!! note

docs/src/search.md

@@ -13,11 +13,15 @@ A vector search finds the approximate or exact nearest neighbors to a given quer
 Distance metrics are a measure of the similarity between a pair of vectors.
 Currently, LanceDB supports the following metrics:
 
-| Metric   | Description                                                                 |
-| -------- | --------------------------------------------------------------------------- |
-| `l2`     | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `cosine` | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)        |
-| `dot`    | [Dot Production](https://en.wikipedia.org/wiki/Dot_product)                 |
+| Metric    | Description                                                                 |
+| --------- | --------------------------------------------------------------------------- |
+| `l2`      | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
+| `cosine`  | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)        |
+| `dot`     | [Dot Production](https://en.wikipedia.org/wiki/Dot_product)                 |
+| `hamming` | [Hamming Distance](https://en.wikipedia.org/wiki/Hamming_distance)          |
+
+!!! note
+    The `hamming` metric is only available for binary vectors.
 
 ## Exhaustive search (kNN)
 
@@ -40,18 +44,16 @@ db.create_table("my_vectors", data=data)
 
 === "Python"
 
-    ```python
-    import lancedb
-    import numpy as np
+    === "Sync API"
 
-    db = lancedb.connect("data/sample-lancedb")
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search"
+        ```
+    === "Async API"
 
-    tbl = db.open_table("my_vectors")
-
-    df = tbl.search(np.random.random((1536))) \
-        .limit(10) \
-        .to_list()
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_async"
+        ```
 
 === "TypeScript"
 
@@ -77,12 +79,16 @@ By default, `l2` will be used as metric type. You can specify the metric type as
 
 === "Python"
 
-    ```python
-    df = tbl.search(np.random.random((1536))) \
-        .metric("cosine") \
-        .limit(10) \
-        .to_list()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_cosine"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_async_cosine"
+        ```
 
 === "TypeScript"
 
@@ -107,46 +113,92 @@ an ANN search means that using an index often involves a trade-off between recal
 See the [IVF_PQ index](./concepts/index_ivfpq.md) for a deeper description of how `IVF_PQ`
 indexes work in LanceDB.
 
+## Binary vector
+
+LanceDB supports binary vectors as a data type, and has the ability to search binary vectors with hamming distance. The binary vectors are stored as uint8 arrays (every 8 bits are stored as a byte):
+
+!!! note
+    The dim of the binary vector must be a multiple of 8. A vector of dim 128 will be stored as a uint8 array of size 16.
+
+=== "Python"
+
+    === "sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_binary_vector.py:imports"
+
+        --8<-- "python/python/tests/docs/test_binary_vector.py:sync_binary_vector"
+        ```
+
+    === "async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_binary_vector.py:imports"
+
+        --8<-- "python/python/tests/docs/test_binary_vector.py:async_binary_vector"
+        ```
+
+## Search with distance range
+
+You can also search for vectors within a specific distance range from the query vector. This is useful when you want to find vectors that are not just the nearest neighbors, but also those that are within a certain distance. This can be done by using the `distance_range` method.
+
+=== "Python"
+
+    === "sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_distance_range.py:imports"
+
+        --8<-- "python/python/tests/docs/test_distance_range.py:sync_distance_range"
+        ```
+
+    === "async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_distance_range.py:imports"
+
+        --8<-- "python/python/tests/docs/test_distance_range.py:async_distance_range"
+        ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        --8<-- "nodejs/examples/search.test.ts:import"
+
+        --8<-- "nodejs/examples/search.test.ts:distance_range"
+        ```
+
+
 ## Output search results
 
 LanceDB returns vector search results via different formats commonly used in python.
 Let's create a LanceDB table with a nested schema:
 
 === "Python"
+    === "Sync API"
 
-    ```python
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-pydantic"
+        --8<-- "python/python/tests/docs/test_search.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_search.py:import-pydantic-base-model"
+        --8<-- "python/python/tests/docs/test_search.py:class-definition"
+        --8<-- "python/python/tests/docs/test_search.py:create_table_with_nested_schema"
+        ```
+    === "Async API"
 
-    from datetime import datetime
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    import numpy as np
-    from pydantic import BaseModel
-    uri = "data/sample-lancedb-nested"
-
-    class Metadata(BaseModel):
-        source: str
-        timestamp: datetime
-
-    class Document(BaseModel):
-        content: str
-        meta: Metadata
-
-    class LanceSchema(LanceModel):
-        id: str
-        vector: Vector(1536)
-        payload: Document
-
-    # Let's add 100 sample rows to our dataset
-    data = [LanceSchema(
-        id=f"id{i}",
-        vector=np.random.randn(1536),
-        payload=Document(
-            content=f"document{i}", meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now())
-        ),
-    ) for i in range(100)]
-
-    tbl = db.create_table("documents", data=data)
-    ```
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:import-datetime"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
+        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-pydantic"
+        --8<-- "python/python/tests/docs/test_search.py:import-numpy"
+        --8<-- "python/python/tests/docs/test_search.py:import-pydantic-base-model"
+        --8<-- "python/python/tests/docs/test_search.py:class-definition"
+        --8<-- "python/python/tests/docs/test_search.py:create_table_async_with_nested_schema"
+        ```
 
     ### As a PyArrow table
 
@@ -155,17 +207,31 @@ Let's create a LanceDB table with a nested schema:
     the addition of an `_distance` column for vector search or a `score`
     column for full text search.
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_arrow()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pyarrow"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_pyarrow"
+        ```
 
     ### As a Pandas DataFrame
 
     You can also get the results as a pandas dataframe.
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_pandas()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_pandas"
+        ```
 
     While other formats like Arrow/Pydantic/Python dicts have a natural
     way to handle nested schemas, pandas can only store nested data as a
@@ -173,33 +239,50 @@ Let's create a LanceDB table with a nested schema:
     So for convenience, you can also tell LanceDB to flatten a nested schema
     when creating the pandas dataframe.
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas_flatten_true"
+        ```
 
     If your table has a deeply nested struct, you can control how many levels
     of nesting to flatten by passing in a positive integer.
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas_flatten_1"
+        ```
+    !!! note
+        `flatten` is not yet supported with our asynchronous client.
 
     ### As a list of Python dicts
 
     You can of course return results as a list of python dicts.
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_list()
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_list"
+        ```
+    === "Async API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_list"
+        ```
 
     ### As a list of Pydantic models
 
     We can add data using Pydantic models, and we can certainly
     retrieve results as Pydantic models
 
-    ```python
-    tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
-    ```
+    === "Sync API"
+
+        ```python
+        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pydantic"
+        ```
+    !!! note
+        `to_pydantic()` is not yet supported with our asynchronous client.
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.

docs/src/sql.md

@@ -4,8 +4,8 @@
 
 LanceDB supports filtering of query results based on metadata fields. By default, post-filtering is
 performed on the top-k results returned by the vector search. However, pre-filtering is also an
-option that performs the filter prior to vector search. This can be useful to narrow down on
-the search space on a very large dataset to reduce query latency.
+option that performs the filter prior to vector search. This can be useful to narrow down
+the search space of a very large dataset to reduce query latency.
 
 Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
 
@@ -15,13 +15,18 @@ Similarly, a highly selective post-filter can lead to false positives. Increasin
 ```python
 import lancedb
 import numpy as np
+
 uri = "data/sample-lancedb"
-db = lancedb.connect(uri)
-
 data = [{"vector": row, "item": f"item {i}", "id": i}
-     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+    for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
 
+# Synchronous client
+db = lancedb.connect(uri)
 tbl = db.create_table("my_vectors", data=data)
+
+# Asynchronous client
+async_db = await lancedb.connect_async(uri)
+async_tbl = await async_db.create_table("my_vectors_async", data=data)
 ```
 -->
 <!-- Setup Code
@@ -39,13 +44,11 @@ const tbl = await db.createTable('myVectors', data)
 
 === "Python"
 
-    ```py
-    result = (
-        tbl.search([0.5, 0.2])
-        .where("id = 10", prefilter=True)
-        .limit(1)
-        .to_arrow()
-    )
+    ```python
+    # Synchronous client
+    result = tbl.search([0.5, 0.2]).where("id = 10", prefilter=True).limit(1).to_arrow()
+    # Asynchronous client
+    result = await async_tbl.query().where("id = 10").nearest_to([0.5, 0.2]).limit(1).to_arrow()
     ```
 
 === "TypeScript"
@@ -63,15 +66,15 @@ const tbl = await db.createTable('myVectors', data)
         ```
 !!! note
 
-    Creating a [scalar index](guides/scalar_index.md) accelerates filtering
+    Creating a [scalar index](guides/scalar_index.md) accelerates filtering.
 
 ## SQL filters
 
 Because it's built on top of [DataFusion](https://github.com/apache/arrow-datafusion), LanceDB
 embraces the utilization of standard SQL expressions as predicates for filtering operations.
-It can be used during vector search, update, and deletion operations.
+SQL can be used during vector search, update, and deletion operations.
 
-Currently, Lance supports a growing list of SQL expressions.
+LanceDB supports a growing list of SQL expressions:
 
 - `>`, `>=`, `<`, `<=`, `=`
 - `AND`, `OR`, `NOT`
@@ -88,9 +91,17 @@ For example, the following filter string is acceptable:
 === "Python"
 
     ```python
-    tbl.search([100, 102]) \
-       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
-       .to_arrow()
+    # Synchronous client
+    tbl.search([100, 102]).where(
+        "(item IN ('item 0', 'item 2')) AND (id > 10)"
+    ).to_arrow()
+    # Asynchronous client
+    await (
+        async_tbl.query()
+        .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+        .nearest_to([100, 102])
+        .to_arrow()
+    )
     ```
 
 === "TypeScript"
@@ -121,7 +132,7 @@ path must be wrapped in backticks.
 !!!warning "Field names containing periods (`.`) are not supported."
 
 Literals for dates, timestamps, and decimals can be written by writing the string
-value after the type name. For example
+value after the type name. For example:
 
 === "SQL"
 
@@ -163,12 +174,15 @@ The mapping from SQL types to Arrow types is:
 
 ## Filtering without Vector Search
 
-You can also filter your data without search.
+You can also filter your data without search:
 
 === "Python"
 
     ```python
+    # Synchronous client
     tbl.search().where("id = 10").limit(10).to_arrow()
+    # Asynchronous client
+    await async_tbl.query().where("id = 10").limit(10).to_arrow()
     ```
 
 === "TypeScript"

docs/test/md_testing.py

@@ -12,19 +12,23 @@ excluded_globs = [
     "../src/integrations/*.md",
     "../src/guides/tables.md",
     "../src/python/duckdb.md",
+    "../src/python/pandas_and_pyarrow.md",
+    "../src/python/polars_arrow.md",
     "../src/embeddings/*.md",
     "../src/concepts/*.md",
     "../src/ann_indexes.md",
     "../src/basic.md",
+    "../src/search.md",
     "../src/hybrid_search/hybrid_search.md",
     "../src/reranking/*.md",
     "../src/guides/tuning_retrievers/*.md",
     "../src/embeddings/available_embedding_models/text_embedding_functions/*.md",
     "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md",
     "../src/rag/*.md",
-    "../src/rag/advanced_techniques/*.md"
-
-
+    "../src/rag/advanced_techniques/*.md",
+    "../src/guides/scalar_index.md",
+    "../src/guides/storage.md",
+    "../src/search.md"
 ]
 
 python_prefix = "py"

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.14.1-beta.3</version>
+        <version>0.14.2-beta.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.14.1-beta.3</version>
+    <version>0.14.2-beta.0</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.14.1-beta.3",
+  "version": "0.14.2-beta.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.14.1-beta.3",
+      "version": "0.14.2-beta.0",
       "cpu": [
         "x64",
         "arm64"
@@ -52,14 +52,14 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.14.1-beta.3",
-        "@lancedb/vectordb-darwin-x64": "0.14.1-beta.3",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.14.1-beta.3",
-        "@lancedb/vectordb-linux-arm64-musl": "0.14.1-beta.3",
-        "@lancedb/vectordb-linux-x64-gnu": "0.14.1-beta.3",
-        "@lancedb/vectordb-linux-x64-musl": "0.14.1-beta.3",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.14.1-beta.3",
-        "@lancedb/vectordb-win32-x64-msvc": "0.14.1-beta.3"
+        "@lancedb/vectordb-darwin-arm64": "0.14.2-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.14.2-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.14.2-beta.0",
+        "@lancedb/vectordb-linux-arm64-musl": "0.14.2-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.14.2-beta.0",
+        "@lancedb/vectordb-linux-x64-musl": "0.14.2-beta.0",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.14.2-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.14.2-beta.0"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -329,6 +329,97 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.14.2-beta.0.tgz",
+      "integrity": "sha512-nsXOl9M8jhsr/LrfvrVHiuWWj/zX3zU2Aahpw8etjJbnU83nmO1r9agPxN6mD/J60EsLP3gDaiRPaFY66pHScA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-darwin-x64": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.14.2-beta.0.tgz",
+      "integrity": "sha512-E1ouo0EfGaxG26YWnw717vaHGNLulmqzh6eaTQuj45Vd4GaPj07TJygtDyvMFBJdsZjdY5YIc9U8yIem1NfeKQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.14.2-beta.0.tgz",
+      "integrity": "sha512-SewXZLGccZUkONACHHPCW1Z7xsz8MaXifwpaWMEyIzbQBFAIMq30lPZN63bTt/zNo6BcBPv54yz6n1ZfCv5l+w==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-musl": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-musl/-/vectordb-linux-arm64-musl-0.14.2-beta.0.tgz",
+      "integrity": "sha512-ppq3P2QYxPHmikY6nbWTwMhDGP+e+feqzm4iXKhpBxzHR2XwoY5CtDKgKDfEHy1FyCoIyvh2yYT2M1TSkrkOBw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.14.2-beta.0.tgz",
+      "integrity": "sha512-XgkoarmdS42fLMMqNdHTVja2z7a0/Q4h3X+n14Ph/pkYsb7pmOabV4a7+ej8KJPm1wv2GmDA4GXcFPjF0tFBFA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-musl": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-musl/-/vectordb-linux-x64-musl-0.14.2-beta.0.tgz",
+      "integrity": "sha512-vGgUOVb43eccF0oz2YJK+Zionwk4ODelHU7icmGeVsULkkFkoAbf0nO4PY38ZAeLsodnLxHIIu51Bd4Jm9m20w==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.14.2-beta.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.14.2-beta.0.tgz",
+      "integrity": "sha512-zGLC382V3gE1MHQpf0XTe34yiB+6ZtSIuOFMIDEZVI5PVN5XkXULMY6dlt5fvo4IxhRoscGjpmmaNxJzUwigDg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.14.1-beta.3",
+  "version": "0.14.2-beta.0",
   "description": " Serverless, low-latency vector database for AI applications",
   "private": false,
   "main": "dist/index.js",
@@ -92,13 +92,13 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.14.1-beta.3",
-    "@lancedb/vectordb-darwin-arm64": "0.14.1-beta.3",
-    "@lancedb/vectordb-linux-x64-gnu": "0.14.1-beta.3",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.14.1-beta.3",
-    "@lancedb/vectordb-linux-x64-musl": "0.14.1-beta.3",
-    "@lancedb/vectordb-linux-arm64-musl": "0.14.1-beta.3",
-    "@lancedb/vectordb-win32-x64-msvc": "0.14.1-beta.3",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.14.1-beta.3"
+    "@lancedb/vectordb-darwin-x64": "0.14.2-beta.0",
+    "@lancedb/vectordb-darwin-arm64": "0.14.2-beta.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.14.2-beta.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.14.2-beta.0",
+    "@lancedb/vectordb-linux-x64-musl": "0.14.2-beta.0",
+    "@lancedb/vectordb-linux-arm64-musl": "0.14.2-beta.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.14.2-beta.0",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.14.2-beta.0"
   }
 }

nodejs/CONTRIBUTING.md

@@ -0,0 +1,76 @@
+# Contributing to LanceDB Typescript
+
+This document outlines the process for contributing to LanceDB Typescript.
+For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+
+## Project layout
+
+The Typescript package is a wrapper around the Rust library, `lancedb`. We use
+the [napi-rs](https://napi.rs/) library to create the bindings between Rust and
+Typescript.
+
+* `src/`: Rust bindings source code
+* `lancedb/`: Typescript package source code
+* `__test__/`: Unit tests
+* `examples/`: An npm package with the examples shown in the documentation
+
+## Development environment
+
+To set up your development environment, you will need to install the following:
+
+1. Node.js 14 or later
+2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+
+Initial setup:
+
+```shell
+npm install
+```
+
+### Commit Hooks
+
+It is **highly recommended** to install the [pre-commit](https://pre-commit.com/) hooks to ensure that your
+code is formatted correctly and passes basic checks before committing:
+
+```shell
+pre-commit install
+```
+
+## Development
+
+Most common development commands can be run using the npm scripts.
+
+Build the package
+
+```shell
+npm install
+npm run build
+```
+
+Lint:
+
+```shell
+npm run lint
+```
+
+Format and fix lints:
+
+```shell
+npm run lint-fix
+```
+
+Run tests:
+
+```shell
+npm test
+```
+
+To run a single test:
+
+```shell
+# Single file: table.test.ts
+npm test -- table.test.ts
+# Single test: 'merge insert' in table.test.ts
+npm test -- table.test.ts --testNamePattern=merge\ insert
+```

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.14.1-beta.3"
+version = "0.14.2-beta.0"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -12,7 +12,10 @@ categories.workspace = true
 crate-type = ["cdylib"]
 
 [dependencies]
+async-trait.workspace = true
 arrow-ipc.workspace = true
+arrow-array.workspace = true
+arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", features = ["remote"] }

nodejs/README.md

@@ -36,37 +36,4 @@ The [quickstart](../basic.md) contains a more complete example.
 
 ## Development
 
-```sh
-npm run build
-npm run test
-```
-
-### Running lint / format
-
-LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
-To manually lint your code you can run:
-
-```sh
-npm run lint
-```
-
-to automatically fix all fixable issues:
-
-```sh
-npm run lint-fix
-```
-
-If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
-
-### Generating docs
-
-```sh
-npm run docs
-
-cd ../docs
-# Asssume the virtual environment was created
-# python3 -m venv venv
-# pip install -r requirements.txt
-. ./venv/bin/activate
-mkdocs build
-```
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute to LanceDB.

nodejs/__test__/arrow.test.ts

@@ -20,6 +20,8 @@ import * as arrow18 from "apache-arrow-18";
 
 import {
   convertToTable,
+  fromBufferToRecordBatch,
+  fromRecordBatchToBuffer,
   fromTableToBuffer,
   makeArrowTable,
   makeEmptyTable,
@@ -553,5 +555,28 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         });
       });
     });
+
+    describe("converting record batches to buffers", function () {
+      it("can convert to buffered record batch and back again", async function () {
+        const records = [
+          { text: "dog", vector: [0.1, 0.2] },
+          { text: "cat", vector: [0.3, 0.4] },
+        ];
+        const table = await convertToTable(records);
+        const batch = table.batches[0];
+
+        const buffer = await fromRecordBatchToBuffer(batch);
+        const result = await fromBufferToRecordBatch(buffer);
+
+        expect(JSON.stringify(batch.toArray())).toEqual(
+          JSON.stringify(result?.toArray()),
+        );
+      });
+
+      it("converting from buffer returns null if buffer has no record batches", async function () {
+        const result = await fromBufferToRecordBatch(Buffer.from([0x01, 0x02])); // bad data
+        expect(result).toEqual(null);
+      });
+    });
   },
 );

nodejs/__test__/rerankers.test.ts

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { RecordBatch } from "apache-arrow";
+import * as tmp from "tmp";
+import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
+import { RRFReranker } from "../lancedb/rerankers";
+
+describe("rerankers", function () {
+  let tmpDir: tmp.DirResult;
+  let conn: Connection;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    conn = await connect(tmpDir.name);
+    table = await conn.createTable("mytable", [
+      { vector: [0.1, 0.1], text: "dog" },
+      { vector: [0.2, 0.2], text: "cat" },
+    ]);
+    await table.createIndex("text", {
+      config: Index.fts(),
+      replace: true,
+    });
+  });
+
+  it("will query with the custom reranker", async function () {
+    const expectedResult = [
+      {
+        text: "albert",
+        // biome-ignore lint/style/useNamingConvention: this is the lance field name
+        _relevance_score: 0.99,
+      },
+    ];
+    class MyCustomReranker {
+      async rerankHybrid(
+        _query: string,
+        _vecResults: RecordBatch,
+        _ftsResults: RecordBatch,
+      ): Promise<RecordBatch> {
+        // no reranker logic, just return some static data
+        const table = makeArrowTable(expectedResult);
+        return table.batches[0];
+      }
+    }
+
+    let result = await table
+      .query()
+      .nearestTo([0.1, 0.1])
+      .fullTextSearch("dog")
+      .rerank(new MyCustomReranker())
+      .select(["text"])
+      .limit(5)
+      .toArray();
+
+    result = JSON.parse(JSON.stringify(result)); // convert StructRow to Object
+    expect(result).toEqual([
+      {
+        text: "albert",
+        // biome-ignore lint/style/useNamingConvention: this is the lance field name
+        _relevance_score: 0.99,
+      },
+    ]);
+  });
+
+  it("will query with RRFReranker", async function () {
+    // smoke test to see if the Rust wrapping Typescript is wired up correctly
+    const result = await table
+      .query()
+      .nearestTo([0.1, 0.1])
+      .fullTextSearch("dog")
+      .rerank(await RRFReranker.create())
+      .select(["text"])
+      .limit(5)
+      .toArray();
+
+    expect(result).toHaveLength(2);
+  });
+});

nodejs/__test__/table.test.ts

@@ -475,6 +475,62 @@ describe("When creating an index", () => {
     expect(rst.numRows).toBe(1);
   });
 
+  it("should search with distance range", async () => {
+    await tbl.createIndex("vec");
+
+    const rst = await tbl.query().limit(10).nearestTo(queryVec).toArrow();
+    const distanceColumn = rst.getChild("_distance");
+    let minDist = undefined;
+    let maxDist = undefined;
+    if (distanceColumn) {
+      minDist = distanceColumn.get(0);
+      maxDist = distanceColumn.get(9);
+    }
+
+    const rst2 = await tbl
+      .query()
+      .limit(10)
+      .nearestTo(queryVec)
+      .distanceRange(minDist, maxDist)
+      .toArrow();
+    const distanceColumn2 = rst2.getChild("_distance");
+    expect(distanceColumn2).toBeDefined();
+    if (distanceColumn2) {
+      for await (const d of distanceColumn2) {
+        expect(d).toBeGreaterThanOrEqual(minDist);
+        expect(d).toBeLessThan(maxDist);
+      }
+    }
+
+    const rst3 = await tbl
+      .query()
+      .limit(10)
+      .nearestTo(queryVec)
+      .distanceRange(maxDist, undefined)
+      .toArrow();
+    const distanceColumn3 = rst3.getChild("_distance");
+    expect(distanceColumn3).toBeDefined();
+    if (distanceColumn3) {
+      for await (const d of distanceColumn3) {
+        expect(d).toBeGreaterThanOrEqual(maxDist);
+      }
+    }
+
+    const rst4 = await tbl
+      .query()
+      .limit(10)
+      .nearestTo(queryVec)
+      .distanceRange(undefined, minDist)
+      .toArrow();
+    const distanceColumn4 = rst4.getChild("_distance");
+    expect(distanceColumn4).toBeDefined();
+    if (distanceColumn4) {
+      for await (const d of distanceColumn4) {
+        expect(d).toBeLessThan(minDist);
+      }
+    }
+  });
+
   it("should create and search IVF_HNSW indices", async () => {
     await tbl.createIndex("vec", {
       config: Index.hnswSq(),

nodejs/examples/search.test.ts

@@ -38,5 +38,19 @@ test("full text search", async () => {
       .toArray();
     // --8<-- [end:search2]
     expect(results2.length).toBe(10);
+
+    // --8<-- [start:distance_range]
+    const results3 = await (
+      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
+    )
+      .distanceType("cosine")
+      .distanceRange(0.1, 0.2)
+      .limit(10)
+      .toArray();
+    // --8<-- [end:distance_range]
+    for (const r of results3) {
+      expect(r.distance).toBeGreaterThanOrEqual(0.1);
+      expect(r.distance).toBeLessThan(0.2);
+    }
   });
 });

nodejs/lancedb/arrow.ts

@@ -27,7 +27,9 @@ import {
   List,
   Null,
   RecordBatch,
+  RecordBatchFileReader,
   RecordBatchFileWriter,
+  RecordBatchReader,
   RecordBatchStreamWriter,
   Schema,
   Struct,
@@ -810,6 +812,30 @@ export async function fromDataToBuffer(
   }
 }
 
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+export async function fromBufferToRecordBatch(
+  data: Buffer,
+): Promise<RecordBatch | null> {
+  const iter = await RecordBatchFileReader.readAll(Buffer.from(data)).next()
+    .value;
+  const recordBatch = iter?.next().value;
+  return recordBatch || null;
+}
+
+/**
+ * Create a buffer containing a single record batch
+ */
+export async function fromRecordBatchToBuffer(
+  batch: RecordBatch,
+): Promise<Buffer> {
+  const writer = new RecordBatchFileWriter().writeAll([batch]);
+  return Buffer.from(await writer.toUint8Array());
+}
+
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

nodejs/lancedb/index.ts

@@ -62,6 +62,7 @@ export { Index, IndexOptions, IvfPqOptions } from "./indices";
 export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 
 export * as embedding from "./embedding";
+export * as rerankers from "./rerankers";
 
 /**
  * Connect to a LanceDB instance at the given URI.

nodejs/lancedb/query.ts

@@ -16,6 +16,8 @@ import {
   Table as ArrowTable,
   type IntoVector,
   RecordBatch,
+  fromBufferToRecordBatch,
+  fromRecordBatchToBuffer,
   tableFromIPC,
 } from "./arrow";
 import { type IvfPqOptions } from "./indices";
@@ -25,6 +27,7 @@ import {
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
+import { Reranker } from "./rerankers";
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -385,6 +388,19 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
     return this;
   }
 
+  /*
+   * Set the distance range to use
+   *
+   * Only rows with distances within range [lower_bound, upper_bound)
+   * will be returned.
+   *
+   * `undefined` means no lower or upper bound.
+   */
+  distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
+    super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
+    return this;
+  }
+
   /**
    * Set the number of candidates to consider during the search
    *
@@ -542,6 +558,27 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
       return this;
     }
   }
+
+  rerank(reranker: Reranker): VectorQuery {
+    super.doCall((inner) =>
+      inner.rerank({
+        rerankHybrid: async (_, args) => {
+          const vecResults = await fromBufferToRecordBatch(args.vecResults);
+          const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
+          const result = await reranker.rerankHybrid(
+            args.query,
+            vecResults as RecordBatch,
+            ftsResults as RecordBatch,
+          );
+
+          const buffer = fromRecordBatchToBuffer(result);
+          return buffer;
+        },
+      }),
+    );
+
+    return this;
+  }
 }
 
 /** A builder for LanceDB queries. */

nodejs/lancedb/rerankers/index.ts

@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { RecordBatch } from "apache-arrow";
+
+export * from "./rrf";
+
+// Interface for a reranker. A reranker is used to rerank the results from a
+// vector and FTS search. This is useful for combining the results from both
+// search methods.
+export interface Reranker {
+  rerankHybrid(
+    query: string,
+    vecResults: RecordBatch,
+    ftsResults: RecordBatch,
+  ): Promise<RecordBatch>;
+}

nodejs/lancedb/rerankers/rrf.ts

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { RecordBatch } from "apache-arrow";
+import { fromBufferToRecordBatch, fromRecordBatchToBuffer } from "../arrow";
+import { RrfReranker as NativeRRFReranker } from "../native";
+
+/**
+ * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
+ *
+ * Internally this uses the Rust implementation
+ */
+export class RRFReranker {
+  private inner: NativeRRFReranker;
+
+  constructor(inner: NativeRRFReranker) {
+    this.inner = inner;
+  }
+
+  public static async create(k: number = 60) {
+    return new RRFReranker(
+      await NativeRRFReranker.tryNew(new Float32Array([k])),
+    );
+  }
+
+  async rerankHybrid(
+    query: string,
+    vecResults: RecordBatch,
+    ftsResults: RecordBatch,
+  ): Promise<RecordBatch> {
+    const buffer = await this.inner.rerankHybrid(
+      query,
+      await fromRecordBatchToBuffer(vecResults),
+      await fromRecordBatchToBuffer(ftsResults),
+    );
+    const recordBatch = await fromBufferToRecordBatch(buffer);
+
+    return recordBatch as RecordBatch;
+  }
+}

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.14.1-beta.3",
+  "version": "0.14.2-beta.0",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.14.1-beta.3",
+	"version": "0.14.2-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.14.0",
+  "version": "0.14.2-beta.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.14.0",
+      "version": "0.14.2-beta.0",
       "cpu": [
         "x64",
         "arm64"
@@ -18,6 +18,7 @@
         "win32"
       ],
       "dependencies": {
+        "@lancedb/lancedb": "^0.14.1",
         "reflect-metadata": "^0.2.2"
       },
       "devDependencies": {
@@ -4149,6 +4150,152 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@lancedb/lancedb": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.14.1.tgz",
+      "integrity": "sha512-DfJ887t52n/2s8G1JnzE7gAR4i7UnfP1OjDYnJ4yTk0aIcn76CbVOUegYfURYlYjL+QFdI1MrAzUdMgYgsGGcA==",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "reflect-metadata": "^0.2.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "optionalDependencies": {
+        "@lancedb/lancedb-darwin-arm64": "0.14.1",
+        "@lancedb/lancedb-darwin-x64": "0.14.1",
+        "@lancedb/lancedb-linux-arm64-gnu": "0.14.1",
+        "@lancedb/lancedb-linux-arm64-musl": "0.14.1",
+        "@lancedb/lancedb-linux-x64-gnu": "0.14.1",
+        "@lancedb/lancedb-linux-x64-musl": "0.14.1",
+        "@lancedb/lancedb-win32-arm64-msvc": "0.14.1",
+        "@lancedb/lancedb-win32-x64-msvc": "0.14.1"
+      },
+      "peerDependencies": {
+        "apache-arrow": ">=15.0.0 <=18.1.0"
+      }
+    },
+    "node_modules/@lancedb/lancedb-darwin-arm64": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.14.1.tgz",
+      "integrity": "sha512-eSWV3GydXfyaptPXZ+S3BgXY1YI26oHQDekACaVevRW6/YQD7sS9UhhSZn1mYyDtLTfJu2kOK2XHA9UY8nyuTg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-darwin-x64": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-x64/-/lancedb-darwin-x64-0.14.1.tgz",
+      "integrity": "sha512-ecf50ykF9WCWmpwAjs3Mk2mph7d+rMJ9EVJeX0UJ4KHDC874lnTDo6Tfd9iUcbExtNI1KZbu+CFnYsbQU+R0gw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-arm64-gnu": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.14.1.tgz",
+      "integrity": "sha512-X7ub1fOm7jZ19KFW/u3nDyFvj5XzDPqEVrp9mmcOgSrst3NJEGGBz1JypkLnTWpg/7IpCBs1UO1G7R7LEsHYOA==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-arm64-musl": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.14.1.tgz",
+      "integrity": "sha512-rkiWpsQCXwybwEjcdFXkAeGahiLcK/NQUjZc9WBY6CKk2Y9dICIafYzxZ6MDCY19jeJIgs3JS0mjleUWYr3JFw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-x64-gnu": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.14.1.tgz",
+      "integrity": "sha512-LGp4D58pQJ3+H3GncNxWHkvhIVOKpTzYUBtVfC8he1rwZ6+CiYDyK9Sim/j8o3UJlJ7cP0m3gNUzPfQchQF9WA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-linux-x64-musl": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.14.1.tgz",
+      "integrity": "sha512-V/TeoyKUESPL/8L1z4WLbMFe5ZEv4gtxc0AFK8ghiduFYN/Hckuj4oTo/Y0ysLiBx1At9FCa91hWDB301ibHBg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb-win32-x64-msvc": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.14.1.tgz",
+      "integrity": "sha512-4M8D0j8/3WZv4CKo+Z44sISKPCKWN5MWA0dcEEGw4sEXHF2RJLrMIOOgEpT5NF7VW+X4t2JJxUA6j2T3cXaD8w==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache 2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/@napi-rs/cli": {
       "version": "2.18.3",
       "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.3.tgz",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.14.1-beta.3",
+  "version": "0.14.2-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/lib.rs

@@ -24,6 +24,7 @@ mod iterator;
 pub mod merge;
 mod query;
 pub mod remote;
+mod rerankers;
 mod table;
 mod util;
 

nodejs/src/query.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use lancedb::index::scalar::FullTextSearchQuery;
 use lancedb::query::ExecutableQuery;
 use lancedb::query::Query as LanceDbQuery;
@@ -25,6 +27,8 @@ use napi_derive::napi;
 use crate::error::convert_error;
 use crate::error::NapiErrorExt;
 use crate::iterator::RecordBatchIterator;
+use crate::rerankers::Reranker;
+use crate::rerankers::RerankerCallbacks;
 use crate::util::parse_distance_type;
 
 #[napi]
@@ -167,6 +171,15 @@ impl VectorQuery {
         self.inner = self.inner.clone().nprobes(nprobe as usize);
     }
 
+    #[napi]
+    pub fn distance_range(&mut self, lower_bound: Option<f64>, upper_bound: Option<f64>) {
+        // napi doesn't support f32, so we have to convert to f32
+        self.inner = self
+            .inner
+            .clone()
+            .distance_range(lower_bound.map(|v| v as f32), upper_bound.map(|v| v as f32));
+    }
+
     #[napi]
     pub fn ef(&mut self, ef: u32) {
         self.inner = self.inner.clone().ef(ef as usize);
@@ -218,6 +231,14 @@ impl VectorQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
+    #[napi]
+    pub fn rerank(&mut self, callbacks: RerankerCallbacks) {
+        self.inner = self
+            .inner
+            .clone()
+            .rerank(Arc::new(Reranker::new(callbacks)));
+    }
+
     #[napi(catch_unwind)]
     pub async fn execute(
         &self,

nodejs/src/rerankers.rs

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use arrow_array::RecordBatch;
+use async_trait::async_trait;
+use napi::{
+    bindgen_prelude::*,
+    threadsafe_function::{ErrorStrategy, ThreadsafeFunction},
+};
+use napi_derive::napi;
+
+use lancedb::ipc::batches_to_ipc_file;
+use lancedb::rerankers::Reranker as LanceDBReranker;
+use lancedb::{error::Error, ipc::ipc_file_to_batches};
+
+use crate::error::NapiErrorExt;
+
+/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
+/// This contains references to the callbacks that can be used to invoke the
+/// reranking methods on the NodeJS implementation and handles serializing the
+/// record batches to Arrow IPC buffers.
+#[napi]
+pub struct Reranker {
+    /// callback to the Javascript which will call the rerankHybrid method of
+    /// some Reranker implementation
+    rerank_hybrid: ThreadsafeFunction<RerankHybridCallbackArgs, ErrorStrategy::CalleeHandled>,
+}
+
+#[napi]
+impl Reranker {
+    #[napi]
+    pub fn new(callbacks: RerankerCallbacks) -> Self {
+        let rerank_hybrid = callbacks
+            .rerank_hybrid
+            .create_threadsafe_function(0, move |ctx| Ok(vec![ctx.value]))
+            .unwrap();
+
+        Self { rerank_hybrid }
+    }
+}
+
+#[async_trait]
+impl lancedb::rerankers::Reranker for Reranker {
+    async fn rerank_hybrid(
+        &self,
+        query: &str,
+        vector_results: RecordBatch,
+        fts_results: RecordBatch,
+    ) -> lancedb::error::Result<RecordBatch> {
+        let callback_args = RerankHybridCallbackArgs {
+            query: query.to_string(),
+            vec_results: batches_to_ipc_file(&[vector_results])?,
+            fts_results: batches_to_ipc_file(&[fts_results])?,
+        };
+        let promised_buffer: Promise<Buffer> = self
+            .rerank_hybrid
+            .call_async(Ok(callback_args))
+            .await
+            .map_err(|e| Error::Runtime {
+                message: format!("napi error status={}, reason={}", e.status, e.reason),
+            })?;
+        let buffer = promised_buffer.await.map_err(|e| Error::Runtime {
+            message: format!("napi error status={}, reason={}", e.status, e.reason),
+        })?;
+        let mut reader = ipc_file_to_batches(buffer.to_vec())?;
+        let result = reader.next().ok_or(Error::Runtime {
+            message: "reranker result deserialization failed".to_string(),
+        })??;
+
+        return Ok(result);
+    }
+}
+
+impl std::fmt::Debug for Reranker {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("NodeJSRerankerWrapper")
+    }
+}
+
+#[napi(object)]
+pub struct RerankerCallbacks {
+    pub rerank_hybrid: JsFunction,
+}
+
+#[napi(object)]
+pub struct RerankHybridCallbackArgs {
+    pub query: String,
+    pub vec_results: Vec<u8>,
+    pub fts_results: Vec<u8>,
+}
+
+fn buffer_to_record_batch(buffer: Buffer) -> Result<RecordBatch> {
+    let mut reader = ipc_file_to_batches(buffer.to_vec()).default_error()?;
+    reader
+        .next()
+        .ok_or(Error::InvalidInput {
+            message: "expected buffer containing record batch".to_string(),
+        })
+        .default_error()?
+        .map_err(Error::from)
+        .default_error()
+}
+
+/// Wrapper around rust RRFReranker
+#[napi]
+pub struct RRFReranker {
+    inner: lancedb::rerankers::rrf::RRFReranker,
+}
+
+#[napi]
+impl RRFReranker {
+    #[napi]
+    pub async fn try_new(k: &[f32]) -> Result<Self> {
+        let k = k
+            .first()
+            .copied()
+            .ok_or(Error::InvalidInput {
+                message: "must supply RRF Reranker constructor arg 'k'".to_string(),
+            })
+            .default_error()?;
+
+        Ok(Self {
+            inner: lancedb::rerankers::rrf::RRFReranker::new(k),
+        })
+    }
+
+    #[napi]
+    pub async fn rerank_hybrid(
+        &self,
+        query: String,
+        vec_results: Buffer,
+        fts_results: Buffer,
+    ) -> Result<Buffer> {
+        let vec_results = buffer_to_record_batch(vec_results)?;
+        let fts_results = buffer_to_record_batch(fts_results)?;
+
+        let result = self
+            .inner
+            .rerank_hybrid(&query, vec_results, fts_results)
+            .await
+            .unwrap();
+
+        let result_buff = batches_to_ipc_file(&[result]).default_error()?;
+
+        Ok(Buffer::from(result_buff.as_ref()))
+    }
+}

nodejs/src/util.rs

@@ -5,8 +5,9 @@ pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<Dista
         "l2" => Ok(DistanceType::L2),
         "cosine" => Ok(DistanceType::Cosine),
         "dot" => Ok(DistanceType::Dot),
+        "hamming" => Ok(DistanceType::Hamming),
         _ => Err(napi::Error::from_reason(format!(
-            "Invalid distance type '{}'.  Must be one of l2, cosine, or dot",
+            "Invalid distance type '{}'.  Must be one of l2, cosine, dot, or hamming",
             distance_type.as_ref()
         ))),
     }

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.17.1-beta.4"
+current_version = "0.18.0-beta.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/CONTRIBUTING.md

@@ -0,0 +1,78 @@
+# Contributing to LanceDB Python
+
+This document outlines the process for contributing to LanceDB Python.
+For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+
+## Project layout
+
+The Python package is a wrapper around the Rust library, `lancedb`. We use
+[pyo3](https://pyo3.rs/) to create the bindings between Rust and Python.
+
+* `src/`: Rust bindings source code
+* `python/lancedb`: Python package source code
+* `python/tests`: Unit tests
+
+## Development environment
+
+To set up your development environment, you will need to install the following:
+
+1. Python 3.9 or later
+2. Cargo (Rust's package manager). Use [rustup](https://rustup.rs/) to install.
+3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+
+Create a virtual environment to work in:
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip install maturin
+```
+
+### Commit Hooks
+
+It is **highly recommended** to install the pre-commit hooks to ensure that your
+code is formatted correctly and passes basic checks before committing:
+
+```bash
+make develop # this will install pre-commit itself
+pre-commit install
+```
+
+## Development
+
+Most common development commands can be run using the Makefile.
+
+Build the package
+
+```shell
+make develop
+```
+
+Format:
+
+```shell
+make format
+```
+
+Run tests:
+
+```shell
+make test
+make doctest
+```
+
+To run a single test, you can use the `pytest` command directly. Provide the path
+to the test file, and optionally the test name after `::`.
+
+```shell
+# Single file: test_table.py
+pytest -vv python/tests/test_table.py
+# Single test: test_basic in test_table.py
+pytest -vv python/tests/test_table.py::test_basic
+```
+
+To see all commands, run:
+
+```shell
+make help
+```

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.17.1-beta.4"
+version = "0.18.0-beta.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/Makefile

@@ -0,0 +1,32 @@
+PIP_EXTRA_INDEX_URL ?= https://pypi.fury.io/lancedb/
+
+help:		## Show this help.
+	@sed -ne '/@sed/!s/## //p' $(MAKEFILE_LIST)
+
+.PHONY: develop
+develop:	## Install the package in development mode.
+	PIP_EXTRA_INDEX_URL=$(PIP_EXTRA_INDEX_URL) maturin develop --extras tests,dev,embeddings
+
+.PHONY: format
+format:		## Format the code.
+	cargo fmt
+	ruff format python
+
+.PHONY: check
+check:		## Check formatting and lints.
+	cargo fmt --check
+	ruff format --check python
+	cargo clippy
+	ruff check python
+
+.PHONY: fix
+fix:		## Fix python lints
+	ruff check python --fix
+
+.PHONY: doctest
+doctest:	## Run documentation tests.
+	pytest --doctest-modules python/lancedb
+
+.PHONY: test
+test:		## Run tests.
+	pytest python/tests -vv --durations=10 -m "not slow"

python/README.md

@@ -8,6 +8,15 @@ A Python library for [LanceDB](https://github.com/lancedb/lancedb).
 pip install lancedb
 ```
 
+### Preview Releases
+
+Stable releases are created about every 2 weeks. For the latest features and bug fixes, you can install the preview release. These releases receive the same level of testing as stable releases, but are not guaranteed to be available for more than 6 months after they are released. Once your application is stable, we recommend switching to stable releases.
+
+
+```bash
+pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
+```
+
 ## Usage
 
 ### Basic Example
@@ -20,76 +29,6 @@ results = table.search([0.1, 0.3]).limit(20).to_list()
 print(results)
 ```
 
-## Development
+### Development
 
-LanceDb is based on the rust crate `lancedb` and is built with maturin.  In order to build with maturin
-you will either need a conda environment or a virtual environment (venv).
-
-```bash
-python -m venv venv
-. ./venv/bin/activate
-```
-
-Install the necessary packages:
-
-```bash
-python -m pip install .[tests,dev]
-```
-
-To build the python package you can use maturin:
-
-```bash
-# This will build the rust bindings and place them in the appropriate place
-# in your venv or conda environment
-maturin develop
-```
-
-To run the unit tests:
-
-```bash
-pytest
-```
-
-To run the doc tests:
-
-```bash
-pytest --doctest-modules python/lancedb
-```
-
-To run linter and automatically fix all errors:
-
-```bash
-ruff format python
-ruff --fix python
-```
-
-If any packages are missing, install them with:
-
-```bash
-pip install <PACKAGE_NAME>
-```
-
-___
-For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
-
-Activate the virtual environment:
-
-```bash
-. .\venv\Scripts\activate
-```
-
-You may need to run the installs separately:
-
-```bash
-pip install -e .[tests]
-pip install -e .[dev]
-```
-
-`tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
-
-```bash
-pip install wheel
-pip install cargo
-conda install rust
-pip install tantivy
-```
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute to LanceDB.

python/pyproject.toml

@@ -1,9 +1,10 @@
 [project]
 name = "lancedb"
 # version in Cargo.toml
+dynamic = ["version"]
 dependencies = [
     "deprecation",
-    "pylance==0.21.0b3",
+    "pylance==0.21.1b1",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "packaging",
@@ -52,8 +53,9 @@ tests = [
     "pytz",
     "polars>=0.19, <=1.3.0",
     "tantivy",
+    "pyarrow-stubs"
 ]
-dev = ["ruff", "pre-commit"]
+dev = ["ruff", "pre-commit", "pyright"]
 docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
 embeddings = [
@@ -93,3 +95,7 @@ markers = [
     "asyncio",
     "s3_test",
 ]
+
+[tool.pyright]
+include = ["python/lancedb/table.py"]
+pythonVersion = "3.12"

python/python/lancedb/__init__.py

@@ -70,7 +70,7 @@ def connect(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        https://lancedb.github.io/lancedb/guides/storage/
+        <https://lancedb.github.io/lancedb/guides/storage/>
 
     Examples
     --------
@@ -82,11 +82,13 @@ def connect(
 
     For object storage, use a URI prefix:
 
-    >>> db = lancedb.connect("s3://my-bucket/lancedb")
+    >>> db = lancedb.connect("s3://my-bucket/lancedb",
+    ...                      storage_options={"aws_access_key_id": "***"})
 
     Connect to LanceDB cloud:
 
-    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...",
+    ...                      client_config={"retry_config": {"retries": 5}})
 
     Returns
     -------
@@ -164,7 +166,7 @@ async def connect_async(
         default configuration is used.
     storage_options: dict, optional
         Additional options for the storage backend. See available options at
-        https://lancedb.github.io/lancedb/guides/storage/
+        <https://lancedb.github.io/lancedb/guides/storage/>
 
     Examples
     --------

python/python/lancedb/_lancedb.pyi

@@ -1,20 +1,11 @@
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Any, Union, Literal
 
 import pyarrow as pa
 
-class Index:
-    @staticmethod
-    def ivf_pq(
-        distance_type: Optional[str],
-        num_partitions: Optional[int],
-        num_sub_vectors: Optional[int],
-        max_iterations: Optional[int],
-        sample_rate: Optional[int],
-    ) -> Index: ...
-    @staticmethod
-    def btree() -> Index: ...
+from .index import BTree, IvfFlat, IvfPq, Bitmap, LabelList, HnswPq, HnswSq, FTS
 
 class Connection(object):
+    uri: str
     async def table_names(
         self, start_after: Optional[str], limit: Optional[int]
     ) -> list[str]: ...
@@ -42,18 +33,35 @@ class Connection(object):
 class Table:
     def name(self) -> str: ...
     def __repr__(self) -> str: ...
+    def is_open(self) -> bool: ...
+    def close(self) -> None: ...
     async def schema(self) -> pa.Schema: ...
-    async def add(self, data: pa.RecordBatchReader, mode: str) -> None: ...
+    async def add(
+        self, data: pa.RecordBatchReader, mode: Literal["append", "overwrite"]
+    ) -> None: ...
     async def update(self, updates: Dict[str, str], where: Optional[str]) -> None: ...
     async def count_rows(self, filter: Optional[str]) -> int: ...
     async def create_index(
-        self, column: str, config: Optional[Index], replace: Optional[bool]
+        self,
+        column: str,
+        index: Union[IvfFlat, IvfPq, HnswPq, HnswSq, BTree, Bitmap, LabelList, FTS],
+        replace: Optional[bool],
     ): ...
+    async def list_versions(self) -> List[Dict[str, Any]]: ...
     async def version(self) -> int: ...
-    async def checkout(self, version): ...
+    async def checkout(self, version: int): ...
     async def checkout_latest(self): ...
     async def restore(self): ...
-    async def list_indices(self) -> List[IndexConfig]: ...
+    async def list_indices(self) -> list[IndexConfig]: ...
+    async def delete(self, filter: str): ...
+    async def add_columns(self, columns: list[tuple[str, str]]) -> None: ...
+    async def alter_columns(self, columns: list[dict[str, Any]]) -> None: ...
+    async def optimize(
+        self,
+        *,
+        cleanup_since_ms: Optional[int] = None,
+        delete_unverified: Optional[bool] = None,
+    ) -> OptimizeStats: ...
     def query(self) -> Query: ...
     def vector_search(self) -> VectorQuery: ...
 

python/python/lancedb/background_loop.py

@@ -23,3 +23,6 @@ class BackgroundEventLoop:
 
     def run(self, future):
         return asyncio.run_coroutine_threadsafe(future, self.loop).result()
+
+
+LOOP = BackgroundEventLoop()

python/python/lancedb/db.py

@@ -17,12 +17,13 @@ from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
 
-from overrides import EnforceOverrides, override
+from lancedb.embeddings.registry import EmbeddingFunctionRegistry
+from overrides import EnforceOverrides, override  # type: ignore
 
 from lancedb.common import data_to_reader, sanitize_uri, validate_schema
-from lancedb.background_loop import BackgroundEventLoop
+from lancedb.background_loop import LOOP
 
-from ._lancedb import connect as lancedb_connect
+from ._lancedb import connect as lancedb_connect  # type: ignore
 from .table import (
     AsyncTable,
     LanceTable,
@@ -43,8 +44,6 @@ if TYPE_CHECKING:
     from .common import DATA, URI
     from .embeddings import EmbeddingFunctionConfig
 
-LOOP = BackgroundEventLoop()
-
 
 class DBConnection(EnforceOverrides):
     """An active LanceDB connection interface."""
@@ -82,6 +81,10 @@ class DBConnection(EnforceOverrides):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
+        *,
+        storage_options: Optional[Dict[str, str]] = None,
+        data_storage_version: Optional[str] = None,
+        enable_v2_manifest_paths: Optional[bool] = None,
     ) -> Table:
         """Create a [Table][lancedb.table.Table] in the database.
 
@@ -119,6 +122,24 @@ class DBConnection(EnforceOverrides):
             One of "error", "drop", "fill".
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
+        storage_options: dict, optional
+            Additional options for the storage backend. Options already set on the
+            connection will be inherited by the table, but can be overridden here.
+            See available options at
+            <https://lancedb.github.io/lancedb/guides/storage/>
+        data_storage_version: optional, str, default "stable"
+            The version of the data storage format to use. Newer versions are more
+            efficient but require newer versions of lance to read.  The default is
+            "stable" which will use the legacy v2 version.  See the user guide
+            for more details.
+        enable_v2_manifest_paths: bool, optional, default False
+            Use the new V2 manifest paths. These paths provide more efficient
+            opening of datasets with many versions on object stores.  WARNING:
+            turning this on will make the dataset unreadable for older versions
+            of LanceDB (prior to 0.13.0). To migrate an existing dataset, instead
+            use the
+            [Table.migrate_manifest_paths_v2][lancedb.table.Table.migrate_v2_manifest_paths]
+            method.
 
         Returns
         -------
@@ -140,7 +161,7 @@ class DBConnection(EnforceOverrides):
         >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
         ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
         >>> db.create_table("my_table", data)
-        LanceTable(connection=..., name="my_table")
+        LanceTable(name='my_table', version=1, ...)
         >>> db["my_table"].head()
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
@@ -161,7 +182,7 @@ class DBConnection(EnforceOverrides):
         ...    "long": [-122.7, -74.1]
         ... })
         >>> db.create_table("table2", data)
-        LanceTable(connection=..., name="table2")
+        LanceTable(name='table2', version=1, ...)
         >>> db["table2"].head()
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
@@ -184,7 +205,7 @@ class DBConnection(EnforceOverrides):
         ...   pa.field("long", pa.float32())
         ... ])
         >>> db.create_table("table3", data, schema = custom_schema)
-        LanceTable(connection=..., name="table3")
+        LanceTable(name='table3', version=1, ...)
         >>> db["table3"].head()
         pyarrow.Table
         vector: fixed_size_list<item: float>[2]
@@ -218,7 +239,7 @@ class DBConnection(EnforceOverrides):
         ...     pa.field("price", pa.float32()),
         ... ])
         >>> db.create_table("table4", make_batches(), schema=schema)
-        LanceTable(connection=..., name="table4")
+        LanceTable(name='table4', version=1, ...)
 
         """
         raise NotImplementedError
@@ -226,7 +247,13 @@ class DBConnection(EnforceOverrides):
     def __getitem__(self, name: str) -> LanceTable:
         return self.open_table(name)
 
-    def open_table(self, name: str, *, index_cache_size: Optional[int] = None) -> Table:
+    def open_table(
+        self,
+        name: str,
+        *,
+        storage_options: Optional[Dict[str, str]] = None,
+        index_cache_size: Optional[int] = None,
+    ) -> Table:
         """Open a Lance Table in the database.
 
         Parameters
@@ -243,6 +270,11 @@ class DBConnection(EnforceOverrides):
             This cache applies to the entire opened table, across all indices.
             Setting this value higher will increase performance on larger datasets
             at the expense of more RAM
+        storage_options: dict, optional
+            Additional options for the storage backend. Options already set on the
+            connection will be inherited by the table, but can be overridden here.
+            See available options at
+            <https://lancedb.github.io/lancedb/guides/storage/>
 
         Returns
         -------
@@ -309,15 +341,15 @@ class LanceDBConnection(DBConnection):
     >>> db = lancedb.connect("./.lancedb")
     >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
     ...                                   {"vector": [0.5, 1.3], "b": 4}])
-    LanceTable(connection=..., name="my_table")
+    LanceTable(name='my_table', version=1, ...)
     >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
-    LanceTable(connection=..., name="another_table")
+    LanceTable(name='another_table', version=1, ...)
     >>> sorted(db.table_names())
     ['another_table', 'my_table']
     >>> len(db)
     2
     >>> db["my_table"]
-    LanceTable(connection=..., name="my_table")
+    LanceTable(name='my_table', version=1, ...)
     >>> "my_table" in db
     True
     >>> db.drop_table("my_table")
@@ -363,7 +395,7 @@ class LanceDBConnection(DBConnection):
         self._conn = AsyncConnection(LOOP.run(do_connect()))
 
     def __repr__(self) -> str:
-        val = f"{self.__class__.__name__}({self._uri}"
+        val = f"{self.__class__.__name__}(uri={self._uri!r}"
         if self.read_consistency_interval is not None:
             val += f", read_consistency_interval={repr(self.read_consistency_interval)}"
         val += ")"
@@ -403,6 +435,10 @@ class LanceDBConnection(DBConnection):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
+        *,
+        storage_options: Optional[Dict[str, str]] = None,
+        data_storage_version: Optional[str] = None,
+        enable_v2_manifest_paths: Optional[bool] = None,
     ) -> LanceTable:
         """Create a table in the database.
 
@@ -424,12 +460,19 @@ class LanceDBConnection(DBConnection):
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
             embedding_functions=embedding_functions,
+            storage_options=storage_options,
+            data_storage_version=data_storage_version,
+            enable_v2_manifest_paths=enable_v2_manifest_paths,
         )
         return tbl
 
     @override
     def open_table(
-        self, name: str, *, index_cache_size: Optional[int] = None
+        self,
+        name: str,
+        *,
+        storage_options: Optional[Dict[str, str]] = None,
+        index_cache_size: Optional[int] = None,
     ) -> LanceTable:
         """Open a table in the database.
 
@@ -442,7 +485,12 @@ class LanceDBConnection(DBConnection):
         -------
         A LanceTable object representing the table.
         """
-        return LanceTable.open(self, name, index_cache_size=index_cache_size)
+        return LanceTable.open(
+            self,
+            name,
+            storage_options=storage_options,
+            index_cache_size=index_cache_size,
+        )
 
     @override
     def drop_table(self, name: str, ignore_missing: bool = False):
@@ -455,13 +503,7 @@ class LanceDBConnection(DBConnection):
         ignore_missing: bool, default False
             If True, ignore if the table does not exist.
         """
-        try:
-            LOOP.run(self._conn.drop_table(name))
-        except ValueError as e:
-            if not ignore_missing:
-                raise e
-            if f"Table '{name}' was not found" not in str(e):
-                raise e
+        LOOP.run(self._conn.drop_table(name, ignore_missing=ignore_missing))
 
     @override
     def drop_database(self):
@@ -524,6 +566,10 @@ class AsyncConnection(object):
         Any attempt to use the connection after it is closed will result in an error."""
         self._inner.close()
 
+    @property
+    def uri(self) -> str:
+        return self._inner.uri
+
     async def table_names(
         self, *, start_after: Optional[str] = None, limit: Optional[int] = None
     ) -> Iterable[str]:
@@ -557,6 +603,7 @@ class AsyncConnection(object):
         fill_value: Optional[float] = None,
         storage_options: Optional[Dict[str, str]] = None,
         *,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         data_storage_version: Optional[str] = None,
         use_legacy_format: Optional[bool] = None,
         enable_v2_manifest_paths: Optional[bool] = None,
@@ -601,7 +648,7 @@ class AsyncConnection(object):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            https://lancedb.github.io/lancedb/guides/storage/
+            <https://lancedb.github.io/lancedb/guides/storage/>
         data_storage_version: optional, str, default "stable"
             The version of the data storage format to use. Newer versions are more
             efficient but require newer versions of lance to read.  The default is
@@ -730,6 +777,13 @@ class AsyncConnection(object):
         """
         metadata = None
 
+        if embedding_functions is not None:
+            # If we passed in embedding functions explicitly
+            # then we'll override any schema metadata that
+            # may was implicitly specified by the LanceModel schema
+            registry = EmbeddingFunctionRegistry.get_instance()
+            metadata = registry.get_table_metadata(embedding_functions)
+
         # Defining defaults here and not in function prototype.  In the future
         # these defaults will move into rust so better to keep them as None.
         if on_bad_vectors is None:
@@ -791,7 +845,7 @@ class AsyncConnection(object):
             Additional options for the storage backend. Options already set on the
             connection will be inherited by the table, but can be overridden here.
             See available options at
-            https://lancedb.github.io/lancedb/guides/storage/
+            <https://lancedb.github.io/lancedb/guides/storage/>
         index_cache_size: int, default 256
             Set the size of the index cache, specified as a number of entries
 
@@ -822,15 +876,23 @@ class AsyncConnection(object):
         """
         await self._inner.rename_table(old_name, new_name)
 
-    async def drop_table(self, name: str):
+    async def drop_table(self, name: str, *, ignore_missing: bool = False):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
+        ignore_missing: bool, default False
+            If True, ignore if the table does not exist.
         """
-        await self._inner.drop_table(name)
+        try:
+            await self._inner.drop_table(name)
+        except ValueError as e:
+            if not ignore_missing:
+                raise e
+            if f"Table '{name}' was not found" not in str(e):
+                raise e
 
     async def drop_database(self):
         """

python/python/lancedb/embeddings/registry.py

@@ -108,9 +108,14 @@ class EmbeddingFunctionRegistry:
             An empty dict is returned if input is None or does not
             contain b"embedding_functions".
         """
-        if metadata is None or b"embedding_functions" not in metadata:
+        if metadata is None:
+            return {}
+        # Look at both bytes and string keys, since we might use either
+        serialized = metadata.get(
+            b"embedding_functions", metadata.get("embedding_functions")
+        )
+        if serialized is None:
             return {}
-        serialized = metadata[b"embedding_functions"]
         raw_list = json.loads(serialized.decode("utf-8"))
         return {
             obj["vector_column"]: EmbeddingFunctionConfig(