build(python): Add project.dynamic = ["version"] to pyproject.toml

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.1-beta.0"
+current_version = "0.14.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/build_linux_wheel/action.yml

@@ -52,7 +52,12 @@ runs:
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
             && unzip /tmp/protoc.zip -d /usr/local \
             && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -20,7 +20,7 @@ runs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
-        # TODO: pass through interpreter
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -28,7 +28,7 @@ runs:
         args: ${{ inputs.args }}
         docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v3
       with:
         name: windows-wheels
         path: python\target\wheels

.github/workflows/make-release-commit.yml

@@ -43,7 +43,7 @@ on:
 jobs:
   make-release:
     # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
@@ -57,14 +57,15 @@ jobs:
           # trigger any workflows watching for new tags. See:
           # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
           token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-      - name: Validate Lance dependency is at stable version
-        if: ${{ inputs.type == 'stable' }}
-        run: python ci/validate_stable_lance.py
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
       - name: Bump Python version
         if: ${{ inputs.python }}
         working-directory: python

.github/workflows/npm-publish.yml

@@ -334,50 +334,51 @@ jobs:
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  node-windows-arm64:
-    name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-windows-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-win32*.tgz
+  # TODO: https://github.com/lancedb/lancedb/issues/1975
+  # node-windows-arm64:
+  #   name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
+  #   # if: startsWith(github.ref, 'refs/tags/v')
+  #   runs-on: ubuntu-latest
+  #   container: alpine:edge
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       config:
+  #         # - arch: x86_64
+  #         - arch: aarch64
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: Install dependencies
+  #       run: |
+  #         apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+  #         curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
+  #         echo "source $HOME/.cargo/env" >> saved_env
+  #         echo "export CC=clang" >> saved_env
+  #         echo "export AR=llvm-ar" >> saved_env
+  #         source "$HOME/.cargo/env"
+  #         rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
+  #         (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+  #         echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+  #         echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+  #     - name: Configure x86_64 build
+  #       if: ${{ matrix.config.arch == 'x86_64' }}
+  #       run: |
+  #         echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+  #     - name: Configure aarch64 build
+  #       if: ${{ matrix.config.arch == 'aarch64' }}
+  #       run: |
+  #         echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+  #     - name: Build Windows Artifacts
+  #       run: |
+  #         source ./saved_env
+  #         bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
+  #     - name: Upload Windows Artifacts
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: node-native-windows-${{ matrix.config.arch }}
+  #         path: |
+  #           node/dist/lancedb-vectordb-win32*.tgz
 
   nodejs-windows:
     name: lancedb ${{ matrix.target }}
@@ -413,57 +414,58 @@ jobs:
           path: |
             nodejs/dist/*.node
 
-  nodejs-windows-arm64:
-    name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
-    # Only runs on tags that matches the make-release action
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-          printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
-          chmod u+x $HOME/.cargo/bin/cargo-xwin
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
+  # TODO: https://github.com/lancedb/lancedb/issues/1975
+  # nodejs-windows-arm64:
+  #   name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
+  #   # Only runs on tags that matches the make-release action
+  #   # if: startsWith(github.ref, 'refs/tags/v')
+  #   runs-on: ubuntu-latest
+  #   container: alpine:edge
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       config:
+  #         # - arch: x86_64
+  #         - arch: aarch64
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: Install dependencies
+  #       run: |
+  #         apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+  #         curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
+  #         echo "source $HOME/.cargo/env" >> saved_env
+  #         echo "export CC=clang" >> saved_env
+  #         echo "export AR=llvm-ar" >> saved_env
+  #         source "$HOME/.cargo/env"
+  #         rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
+  #         (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
+  #         echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
+  #         echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
+  #         printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
+  #         chmod u+x $HOME/.cargo/bin/cargo-xwin
+  #     - name: Configure x86_64 build
+  #       if: ${{ matrix.config.arch == 'x86_64' }}
+  #       run: |
+  #         echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
+  #     - name: Configure aarch64 build
+  #       if: ${{ matrix.config.arch == 'aarch64' }}
+  #       run: |
+  #         echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
+  #     - name: Build Windows Artifacts
+  #       run: |
+  #         source ./saved_env
+  #         bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
+  #     - name: Upload Windows Artifacts
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: nodejs-native-windows-${{ matrix.config.arch }}
+  #         path: |
+  #           nodejs/dist/*.node
 
   release:
     name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows, node-windows-arm64]
+    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -503,7 +505,7 @@ jobs:
 
   release-nodejs:
     name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64]
+    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')

.github/workflows/pypi-publish.yml

@@ -15,21 +15,15 @@ jobs:
           - platform: x86_64
             manylinux: "2_17"
             extra_args: ""
-            runner: ubuntu-22.04
           - platform: x86_64
             manylinux: "2_28"
             extra_args: "--features fp16kernels"
-            runner: ubuntu-22.04
           - platform: aarch64
-            manylinux: "2_17"
+            manylinux: "2_24"
             extra_args: ""
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: ubuntu-2404-8x-arm64
-          - platform: aarch64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-            runner: ubuntu-2404-8x-arm64
-    runs-on: ${{ matrix.config.runner }}
+          # We don't build fp16 kernels for aarch64, because it uses 
+          # cross compilation image, which doesn't have a new enough compiler.
+    runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/python.yml

@@ -30,10 +30,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.11"
       - name: Install ruff
         run: |
-          pip install ruff==0.8.4
+          pip install ruff==0.5.4
       - name: Format check
         run: ruff format --check .
       - name: Lint

CONTRIBUTING.md

@@ -1,78 +0,0 @@
-# Contributing to LanceDB
-
-LanceDB is an open-source project and we welcome contributions from the community.
-This document outlines the process for contributing to LanceDB.
-
-## Reporting Issues
-
-If you encounter a bug or have a feature request, please open an issue on the
-[GitHub issue tracker](https://github.com/lancedb/lancedb).
-
-## Picking an issue
-
-We track issues on the GitHub issue tracker. If you are looking for something to
-work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
-
-If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
-
-## Configuring Git
-
-First, fork the repository on GitHub, then clone your fork:
-
-```bash
-git clone https://github.com/<username>/lancedb.git
-cd lancedb
-```
-
-Then add the main repository as a remote:
-
-```bash
-git remote add upstream https://github.com/lancedb/lancedb.git
-git fetch upstream
-```
-
-## Setting up your development environment
-
-We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
-
-* [Python](python/CONTRIBUTING.md)
-* [Typescript](nodejs/CONTRIBUTING.md)
-<!-- TODO: add Java contributing guide -->
-* [Documentation](docs/README.md)
-
-
-## Best practices for pull requests
-
-For the best chance of having your pull request accepted, please follow these guidelines:
-
-1. Unit test all bug fixes and new features. Your code will not be merged if it
-   doesn't have tests.
-1. If you change the public API, update the documentation in the `docs` directory.
-1. Aim to minimize the number of changes in each pull request. Keep to solving
-   one problem at a time, when possible.
-1. Before marking a pull request ready-for-review, do a self review of your code.
-   Is it clear why you are making the changes? Are the changes easy to understand?
-1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
-    * New feature: `feat: adding foo API`
-    * Bug fix: `fix: issue with foo API`
-    * Documentation change: `docs: adding foo API documentation`
-1. If your pull request is a work in progress, leave the pull request as a draft.
-   We will assume the pull request is ready for review when it is opened.
-1. When writing tests, test the error cases. Make sure they have understandable
-   error messages.
-
-## Project structure
-
-The core library is written in Rust. The Python, Typescript, and Java libraries
-are wrappers around the Rust library.
-
-* `src/lancedb`: Rust library source code
-* `python`: Python package source code
-* `nodejs`: Typescript package source code
-* `node`: **Deprecated** Typescript package source code
-* `java`: Java package source code
-* `docs`: Documentation source code
-
-## Release process
-
-For information on the release process, see: [release_process.md](release_process.md)

Cargo.toml

@@ -21,16 +21,16 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.23.0", "features" = [
+lance = { "version" = "=0.21.0", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-io = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-index = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-linalg = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-table = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-testing = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-datafusion = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-encoding = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
+], git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-io = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-index = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-linalg = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-table = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-testing = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-datafusion = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
+lance-encoding = { version = "=0.21.0", git = "https://github.com/lancedb/lance.git", tag = "v0.21.0-beta.5" }
 # Note that this one does not include pyarrow
 arrow = { version = "53.2", optional = false }
 arrow-array = "53.2"
@@ -42,8 +42,8 @@ arrow-arith = "53.2"
 arrow-cast = "53.2"
 async-trait = "0"
 chrono = "0.4.35"
-datafusion-common = "44.0"
-datafusion-physical-plan = "44.0"
+datafusion-common = "42.0"
+datafusion-physical-plan = "42.0"
 env_logger = "0.10"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",

ci/sysroot-aarch64-pc-windows-msvc.sh

@@ -53,7 +53,7 @@ curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-42
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
 
-# dbghelp.lib fwpuclnt.lib arm64rt.lib
+# fwpuclnt.lib arm64rt.lib
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
 curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
 
@@ -98,7 +98,7 @@ find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#inclu
 # reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
 # I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
 
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
+(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
 
 (cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
 

ci/validate_stable_lance.py

@@ -1,34 +0,0 @@
-import tomllib
-
-found_preview_lance = False
-
-with open("Cargo.toml", "rb") as f:
-    cargo_data = tomllib.load(f)
-
-    for name, dep in cargo_data["workspace"]["dependencies"].items():
-        if name == "lance" or name.startswith("lance-"):
-            if isinstance(dep, str):
-                version = dep
-            elif isinstance(dep, dict):
-                # Version doesn't have the beta tag in it, so we instead look
-                # at the git tag.
-                version = dep.get('tag', dep.get('version'))
-            else:
-                raise ValueError("Unexpected type for dependency: " + str(dep))
-
-            if "beta" in version:
-                found_preview_lance = True
-                print(f"Dependency '{name}' is a preview version: {version}")
-
-with open("python/pyproject.toml", "rb") as f:
-    py_proj_data = tomllib.load(f)
-
-    for dep in py_proj_data["project"]["dependencies"]:
-        if dep.startswith("pylance"):
-            if "b" in dep:
-                found_preview_lance = True
-                print(f"Dependency '{dep}' is a preview version")
-            break  # Only one pylance dependency
-
-if found_preview_lance:
-    raise ValueError("Found preview version of Lance in dependencies")

docs/README.md

@@ -9,81 +9,36 @@ unreleased features.
 ## Building the docs
 
 ### Setup
-1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
-   Run `make develop` to install the Python package.
-2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
 
-### Preview the docs
+### Building node module and create markdown files
 
-```shell
+See [Javascript docs README](./src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+### Run local server
+
+You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
+
+```bash
 cd docs
 mkdocs serve
 ```
 
-If you want to just generate the HTML files:
+### Run doctest for typescript example
 
-```shell
-PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
-```
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-## Adding examples
-
-To make sure examples are correct, we put examples in test files so they can be
-run as part of our test suites.
-
-You can see the tests are at:
-
-* Python: `python/python/tests/docs`
-* Typescript: `nodejs/examples/`
-
-### Checking python examples
-
-```shell
-cd python
-pytest -vv python/tests/docs
-```
-
-### Checking typescript examples
-
-The `@lancedb/lancedb` package must be built before running the tests:
-
-```shell
-pushd nodejs
-npm ci
+```bash
+cd lancedb/docs
+npm i
 npm run build
-popd
-```
-
-Then you can run the examples by going to the `nodejs/examples` directory and
-running the tests like a normal npm package:
-
-```shell
-pushd nodejs/examples
-npm ci
-npm test
-popd
-```
-
-## API documentation
-
-### Python
-
-The Python API documentation is organized based on the file `docs/src/python/python.md`.
-We manually add entries there so we can control the organization of the reference page.
-**However, this means any new types must be manually added to the file.** No additional
-steps are needed to generate the API documentation.
-
-### Typescript
-
-The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
-
-When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
-The new files should be checked into the repository.
-
-```shell
-pushd nodejs
-npm run docs
-popd
+npm run all
 ```

docs/mkdocs.yml

@@ -146,9 +146,7 @@ nav:
               - Building Custom Rerankers: reranking/custom_reranker.md
               - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
-          - Versioning & Reproducibility: 
-              - sync API: notebooks/reproducibility.ipynb
-              - async API: notebooks/reproducibility_async.ipynb
+          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
           - Migration Guide: migration.md
           - Tuning retrieval performance:
@@ -280,9 +278,7 @@ nav:
           - Building Custom Rerankers: reranking/custom_reranker.md
           - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
-      - Versioning & Reproducibility: 
-          - sync API: notebooks/reproducibility.ipynb
-          - async API: notebooks/reproducibility_async.ipynb
+      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
       - Migration Guide: migration.md
       - Tuning retrieval performance:

docs/src/ann_indexes.md

@@ -18,24 +18,25 @@ See the [indexing](concepts/index_ivfpq.md) concepts guide for more information
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
-    === "Sync API"
 
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
-        ```
-    === "Async API"
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    ```python
+    import lancedb
+    import numpy as np
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
-        ```
+    # Create 10,000 sample vectors
+    data = [{"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+    # Add the vectors to a table
+    tbl = db.create_table("my_vectors", data=data)
+
+    # Create and train the index - you need to have enough data in the table for an effective training step
+    tbl.create_index(num_partitions=256, num_sub_vectors=96)
+    ```
 
 === "TypeScript"
 
@@ -126,9 +127,7 @@ You can specify the GPU device to train IVF partitions via
         accelerator="mps"
     )
     ```
-!!! note
-    GPU based indexing is not yet supported with our asynchronous client.
-    
+
 Troubleshooting:
 
 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
@@ -153,16 +152,14 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
-        ```
+    ```python
+    tbl.search(np.random.random((1536))) \
+        .limit(2) \
+        .nprobes(20) \
+        .refine_factor(10) \
+        .to_pandas()
+    ```
 
     ```text
                                               vector       item       _distance
@@ -199,16 +196,10 @@ The search will return the data requested in addition to the distance of each it
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
-        ```
+    ```python
+    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+    ```
 
 === "TypeScript"
 
@@ -230,16 +221,10 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+    ```
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
-        ```
 
     ```text
                                                 vector _distance

docs/src/concepts/index_hnsw.md

@@ -7,7 +7,7 @@ Approximate Nearest Neighbor (ANN) search is a method for finding data points ne
 There are three main types of ANN search algorithms:
 
 * **Tree-based search algorithms**: Use a tree structure to organize and store data points.
-* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
+* * **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
 * **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 
 
 HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  

docs/src/fts.md

@@ -10,20 +10,28 @@ LanceDB provides support for full-text search via Lance, allowing you to incorpo
 Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
-        ```
-    === "Async API"
+    ```python
+    import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
-        ```
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table",
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+    )
+
+    # passing `use_tantivy=False` to use lance FTS index
+    # `use_tantivy=True` by default
+    table.create_fts_index("text", use_tantivy=False)
+    table.search("puppy").limit(10).select(["text"]).to_list()
+    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+    # ...
+    ```
 
 === "TypeScript"
 
@@ -42,7 +50,7 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
     });
 
     await tbl
-        .search("puppy", "fts")
+        .search("puppy", queryType="fts")
         .select(["text"])
         .limit(10)
         .toArray();
@@ -85,32 +93,22 @@ By default the text is tokenized by splitting on punctuation and whitespaces, an
 Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.
 
 For example, to enable stemming for English:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
-    ```
+```python
+table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
+```
 
 the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
 
 The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.
 
 For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
-    ```
+```python
+table.create_fts_index("text",
+                        use_tantivy=False,
+                        language="French",
+                        stem=True,
+                        ascii_folding=True)
+```
 
 ## Filtering
 
@@ -121,16 +119,9 @@ This can be invoked via the familiar `where` syntax.
 With pre-filtering:
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
-        ```
+    ```python
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=True).to_list()
+    ```
 
 === "TypeScript"
 
@@ -160,16 +151,9 @@ With pre-filtering:
 With post-filtering:
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
-        ```
+    ```python
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=False).to_list()
+    ```
 
 === "TypeScript"
 
@@ -207,16 +191,9 @@ or a **terms** search query like `old man sea`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
 
 To search for a phrase, the index must be created with `with_position=True`:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
-    ```
+```python
+table.create_fts_index("text", use_tantivy=False, with_position=True)
+```
 This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.
 
 
@@ -228,16 +205,10 @@ This can make the query more efficient, especially when the table is large and t
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
-        ```
+    ```python
+    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    table.optimize()
+    ```
 
 === "TypeScript"
 

docs/src/fts_tantivy.md

@@ -2,7 +2,7 @@
 
 LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
 
-The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
+The tantivy-based FTS is only available in Python and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
 
 ## Installation
 

docs/src/guides/scalar_index.md

@@ -32,20 +32,19 @@ over scalar columns.
 ### Create a scalar index
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    books = [
+      {"book_id": 1, "publisher": "plenty of books", "tags": ["fantasy", "adventure"]},
+      {"book_id": 2, "publisher": "book town", "tags": ["non-fiction"]},
+      {"book_id": 3, "publisher": "oreilly", "tags": ["textbook"]}
+    ]
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
-        ```
+    db = lancedb.connect("./db")
+    table = db.create_table("books", books)
+    table.create_scalar_index("book_id")  # BTree by default
+    table.create_scalar_index("publisher", index_type="BITMAP")
+    ```
 
 === "Typescript"
 
@@ -63,18 +62,12 @@ The following scan will be faster if the column `book_id` has a scalar index:
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
-        ```
+    table = db.open_table("books")
+    my_df = table.search().where("book_id = 2").to_pandas()
+    ```
 
 === "Typescript"
 
@@ -95,18 +88,22 @@ Scalar indices can also speed up scans containing a vector search or full text s
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
-        ```
-    === "Async API"
+    data = [
+      {"book_id": 1, "vector": [1, 2]},
+      {"book_id": 2, "vector": [3, 4]},
+      {"book_id": 3, "vector": [5, 6]}
+    ]
+    table = db.create_table("book_with_embeddings", data)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
-        ```
+    (
+        table.search([1, 2])
+        .where("book_id != 3", prefilter=True)
+        .to_pandas()
+    )
+    ```
 
 === "Typescript"
 
@@ -125,16 +122,10 @@ Scalar indices can also speed up scans containing a vector search or full text s
 Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
-        ```
+    ```python
+    table.add([{"vector": [7, 8], "book_id": 4}])
+    table.optimize()
+    ```
 
 === "TypeScript"
 

docs/src/guides/storage.md

@@ -12,50 +12,26 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 === "Python"
 
     AWS S3:
-    === "Sync API"
 
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("s3://bucket/path")
+    ```
 
     Google Cloud Storage:
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("gs://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("gs://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("gs://bucket/path")
+    ```
 
     Azure Blob Storage:
 
     <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("az://bucket/path")
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("az://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("az://bucket/path")
+    ```
     Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
 
 
@@ -118,24 +94,13 @@ If you only want this to apply to one particular connection, you can pass the `s
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"}
+    )
+    ```
 
 === "TypeScript"
 
@@ -163,29 +128,15 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        table = db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        async_table = await async_db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async("s3://bucket/path")
+    table = await db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"}
+    )
+    ```
 
 === "TypeScript"
 
@@ -243,32 +194,17 @@ These can be set as environment variables or passed in the `storage_options` par
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "aws_access_key_id": "my-access-key",
+            "aws_secret_access_key": "my-secret-key",
+            "aws_session_token": "my-session-token",
+        }
+    )
+    ```
 
 === "TypeScript"
 
@@ -412,22 +348,12 @@ name of the table to use.
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```
 
 === "JavaScript"
 
@@ -515,30 +441,16 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        }
+    )
+    ```
 
 === "TypeScript"
 
@@ -590,30 +502,16 @@ To configure LanceDB to use an S3 Express endpoint, you must set the storage opt
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        }
+    )
+    ```
 
 === "TypeScript"
 
@@ -654,29 +552,15 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        }
+    )
+    ```
 
 === "TypeScript"
 
@@ -728,31 +612,16 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
 === "Python"
 
     <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )   
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "az://my-container/my-database",
+        storage_options={
+            account_name: "some-account",
+            account_key: "some-key",
+        }
+    )
+    ```
 
 === "TypeScript"
 

docs/src/guides/tables.md

@@ -12,18 +12,10 @@ Initialize a LanceDB connection and create a table
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:connect"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:connect_async"
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("./.lancedb")
+    ```
 
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
@@ -55,16 +47,18 @@ Initialize a LanceDB connection and create a table
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table"
-        ```
-    === "Async API"
+    db = lancedb.connect("./.lancedb")
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async"
-        ```
+    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
+
+    db.create_table("my_table", data)
+
+    db["my_table"].head()
+    ```
 
     !!! info "Note"
         If the table already exists, LanceDB will raise an error by default.
@@ -73,30 +67,16 @@ Initialize a LanceDB connection and create a table
         and the table exists, then it simply opens the existing table. The data you
         passed in will NOT be appended to the table in that case.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_exist_ok"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_exist_ok"
-        ```
+    ```python
+    db.create_table("name", data, exist_ok=True)
+    ```
 
     Sometimes you want to make sure that you start fresh. If you want to
     overwrite the table, you can pass in mode="overwrite" to the createTable function.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_overwrite"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_overwrite"
-        ```
+    ```python
+    db.create_table("name", data, mode="overwrite")
+    ```
 
 === "Typescript[^1]"
     You can create a LanceDB table in JavaScript using an array of records as follows.
@@ -166,37 +146,34 @@ Initialize a LanceDB connection and create a table
 
 ### From a Pandas DataFrame
 
+```python
+import pandas as pd
 
-=== "Sync API"
+data = pd.DataFrame({
+    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+    "lat": [45.5, 40.1],
+    "long": [-122.7, -74.1]
+})
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pandas"
-    ```
-=== "Async API"
+db.create_table("my_table", data)
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pandas"
-    ```
+db["my_table"].head()
+```
 
 !!! info "Note"
     Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
 The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.
 
-=== "Sync API"
+```python
+custom_schema = pa.schema([
+pa.field("vector", pa.list_(pa.float32(), 4)),
+pa.field("lat", pa.float32()),
+pa.field("long", pa.float32())
+])
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_custom_schema"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_custom_schema"
-    ```
+table = db.create_table("my_table", data, schema=custom_schema)
+```
 
 ### From a Polars DataFrame
 
@@ -205,38 +182,45 @@ written in Rust. Just like in Pandas, the Polars integration is enabled by PyArr
 under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
 is on the way.
 
-=== "Sync API"
+```python
+import polars as pl
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_polars"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_polars"
-    ```
+data = pl.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pl_table", data=data)
+```
 
 ### From an Arrow Table
 You can also create LanceDB tables directly from Arrow tables.
 LanceDB supports float16 data type!
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_arrow_table"
-        ```
-    === "Async API"
+    ```python
+    import pyarrows as pa
+    import numpy as np
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_arrow_table"
-        ```
+    dim = 16
+    total = 2
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float16(), dim)),
+            pa.field("text", pa.string())
+        ]
+    )
+    data = pa.Table.from_arrays(
+        [
+            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
+                    pa.list_(pa.float16(), dim)),
+            pa.array(["foo", "bar"])
+        ],
+        ["vector", "text"],
+    )
+    tbl = db.create_table("f16_tbl", data, schema=schema)
+    ```
 
 === "Typescript[^1]"
 
@@ -266,22 +250,25 @@ can be configured with the vector dimensions. It is also important to note that
 LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
 (which itself derives from `pydantic.BaseModel`).
 
-=== "Sync API"
+```python
+from lancedb.pydantic import Vector, LanceModel
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pydantic"
-    ```
-=== "Async API"
+class Content(LanceModel):
+    movie_id: int
+    vector: Vector(128)
+    genres: str
+    title: str
+    imdb_id: int
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pydantic"
-    ```
+    @property
+    def imdb_url(self) -> str:
+        return f"https://www.imdb.com/title/tt{self.imdb_id}"
+
+import pyarrow as pa
+db = lancedb.connect("~/.lancedb")
+table_name = "movielens_small"
+table = db.create_table(table_name, schema=Content)
+```
 
 #### Nested schemas
 
@@ -290,24 +277,22 @@ For example, you may want to store the document string
 and the document source name as a nested Document object:
 
 ```python
---8<-- "python/python/tests/docs/test_guide_tables.py:import-pydantic-basemodel"
---8<-- "python/python/tests/docs/test_guide_tables.py:class-Document"
+class Document(BaseModel):
+    content: str
+    source: str
 ```
 
 This can be used as the type of a LanceDB table column:
 
-=== "Sync API"
+```python
+class NestedSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    document: Document
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_nested_schema"
-    ```
-=== "Async API"
+tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+```
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_nested_schema"
-    ```
 This creates a struct column called "document" that has two subfields
 called "content" and "source":
 
@@ -371,20 +356,29 @@ LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generat
 
 Here's an example using using `RecordBatch` iterator for creating tables.
 
-=== "Sync API"
+```python
+import pyarrow as pa
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_batch"
-    ```
-=== "Async API"
+def make_batches():
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                        pa.list_(pa.float32(), 4)),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"],
+        )
 
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_batch"
-    ```
+schema = pa.schema([
+    pa.field("vector", pa.list_(pa.float32(), 4)),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+db.create_table("batched_tale", make_batches(), schema=schema)
+```
 
 You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
@@ -393,29 +387,15 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 === "Python"
     If you forget the name of your table, you can always get a listing of all table names.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables_async"
-        ```
+    ```python
+    print(db.table_names())
+    ```
 
     Then, you can open any existing tables.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table_async"
-        ```
+    ```python
+    tbl = db.open_table("my_table")
+    ```
 
 === "Typescript[^1]"
 
@@ -438,41 +418,35 @@ You can create an empty table for scenarios where you want to add data to the ta
 
 
     An empty table can be initialized via a PyArrow schema.
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table"
-        ```
-    === "Async API"
+    ```python
+    import lancedb
+    import pyarrow as pa
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async"
-        ```
+    schema = pa.schema(
+      [
+          pa.field("vector", pa.list_(pa.float32(), 2)),
+          pa.field("item", pa.string()),
+          pa.field("price", pa.float32()),
+      ])
+    tbl = db.create_table("empty_table_add", schema=schema)
+    ```
 
     Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
     directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
     that has been extended to support LanceDB specific types like `Vector`.
 
-    === "Sync API"
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, vector
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_pydantic"
-        ```
-    === "Async API"
+    class Item(LanceModel):
+        vector: Vector(2)
+        item: str
+        price: float
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async_pydantic"
-        ```
+    tbl = db.create_table("empty_table_add", schema=Item.to_arrow_schema())
+    ```
 
     Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
@@ -499,96 +473,86 @@ After a table has been created, you can always add more data to it using the `ad
 
     ### Add a Pandas DataFrame
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pandas"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pandas"
-        ```
+    ```python
+    df = pd.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
+    })
+    tbl.add(df)
+    ```
 
     ### Add a Polars DataFrame
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_polars"
-        ```
-    === "Async API"
-    
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_polars"
-        ```
+    ```python
+    df = pl.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
+    })
+    tbl.add(df)
+    ```
 
     ### Add an Iterator
 
     You can also add a large dataset batch in one go using Iterator of any supported data types.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_batch"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_batch"
-        ```
+    ```python
+    def make_batches():
+        for i in range(5):
+            yield [
+                    {"vector": [3.1, 4.1], "item": "peach", "price": 6.0},
+                    {"vector": [5.9, 26.5], "item": "pear", "price": 5.0}
+                ]
+    tbl.add(make_batches())
+    ```
 
     ### Add a PyArrow table
 
     If you have data coming in as a PyArrow table, you can add it directly to the LanceDB table.
 
-    === "Sync API"
+    ```python
+    pa_table = pa.Table.from_arrays(
+            [
+                pa.array([[9.1, 6.7], [9.9, 31.2]],
+                        pa.list_(pa.float32(), 2)),
+                pa.array(["mango", "orange"]),
+                pa.array([7.0, 4.0]),
+            ],
+            ["vector", "item", "price"],
+        )
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pyarrow"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pyarrow"
-        ```
+    tbl.add(pa_table)
+    ```
 
     ### Add a Pydantic Model
 
     Assuming that a table has been created with the correct schema as shown [above](#creating-empty-table), you can add data items that are valid Pydantic models to the table.
 
-    === "Sync API"
+    ```python
+    pydantic_model_items = [
+        Item(vector=[8.1, 4.7], item="pineapple", price=10.0),
+        Item(vector=[6.9, 9.3], item="avocado", price=9.0)
+    ]
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pydantic"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pydantic"
-        ```
+    tbl.add(pydantic_model_items)
+    ```
 
     ??? "Ingesting Pydantic models with LanceDB embedding API"
         When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` field as None to allow LanceDB to automatically vectorize the data.
 
-        === "Sync API"
+        ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import get_registry
 
-            ```python
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_with_embedding"
-            ```
-        === "Async API"
+        db = lancedb.connect("~/tmp")
+        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
 
-            ```python
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_with_embedding"
-            ```
+        class Schema(LanceModel):
+            text: str = embed_fcn.SourceField()
+            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
+
+        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
+        models = [Schema(text="hello"), Schema(text="world")]
+        tbl.add(models)
+        ```
 
 === "Typescript[^1]"
 
@@ -607,41 +571,44 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row_async"
-        ```
+    ```python
+    tbl.delete('item = "fizz"')
+    ```
 
     ### Deleting row with specific column value
 
-    === "Sync API"
+    ```python
+    import lancedb
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row"
-        ```
-    === "Async API"
+    data = [{"x": 1, "vector": [1, 2]},
+            {"x": 2, "vector": [3, 4]},
+            {"x": 3, "vector": [5, 6]}]
+    db = lancedb.connect("./.lancedb")
+    table = db.create_table("my_table", data)
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  2  [3.0, 4.0]
+    # 2  3  [5.0, 6.0]
+
+    table.delete("x = 2")
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  3  [5.0, 6.0]
+    ```
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row_async"
-        ```
-    
     ### Delete from a list of values
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values"
-        ```
-    === "Async API"
+    ```python
+    to_remove = [1, 5]
+    to_remove = ", ".join(str(v) for v in to_remove)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values_async"
-        ```
+    table.delete(f"x IN ({to_remove})")
+    table.to_pandas()
+    #   x      vector
+    # 0  3  [5.0, 6.0]
+    ```
 
 === "Typescript[^1]"
 
@@ -692,20 +659,27 @@ This can be used to update zero to all rows depending on how many rows match the
 === "Python"
 
     API Reference: [lancedb.table.Table.update][]
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table"
-        ```
-    === "Async API"
+    ```python
+    import lancedb
+    import pandas as pd
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_async"
-        ```
+    # Create a lancedb connection
+    db = lancedb.connect("./.lancedb")
+
+    # Create a table from a pandas DataFrame
+    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    table = db.create_table("my_table", data)
+
+    # Update the table where x = 2
+    table.update(where="x = 2", values={"vector": [10, 10]})
+
+    # Get the updated table as a pandas DataFrame
+    df = table.to_pandas()
+
+    # Print the DataFrame
+    print(df)
+    ```
 
     Output
     ```shell
@@ -760,16 +734,13 @@ This can be used to update zero to all rows depending on how many rows match the
   The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql"
-        ```
-    === "Async API"
+    ```python
+    # Update the table where x = 2
+    table.update(valuesSql={"x": "x + 1"})
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql_async"
-        ```
+    print(table.to_pandas())
+    ```
 
     Output
     ```shell
@@ -800,16 +771,11 @@ This can be used to update zero to all rows depending on how many rows match the
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-        ```
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
       By default, if the table does not exist an exception is raised. To suppress this,
@@ -843,16 +809,9 @@ data type for it.
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_columns_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:add_columns"
+    ```
     **API Reference:** [lancedb.table.Table.add_columns][]
 
 === "Typescript"
@@ -889,18 +848,10 @@ rewriting the column, which can be a heavy operation.
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_basic.py:alter_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_basic.py:alter_columns_async"
-        ```
+    ```python
+    import pyarrow as pa
+    --8<-- "python/python/tests/docs/test_basic.py:alter_columns"
+    ```
     **API Reference:** [lancedb.table.Table.alter_columns][]
 
 === "Typescript"
@@ -921,16 +872,9 @@ will remove the column from the schema.
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_columns_async"
-        ```
+    ```python
+     --8<-- "python/python/tests/docs/test_basic.py:drop_columns"
+    ```
     **API Reference:** [lancedb.table.Table.drop_columns][]
 
 === "Typescript"
@@ -981,46 +925,31 @@ There are three possible settings for `read_consistency_interval`:
 
     To set strong consistency, use `timedelta(0)`:
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_strong_consistency"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_strong_consistency"
-        ```
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
+    table = db.open_table("my_table")
+    ```
 
     For eventual consistency, use a custom `timedelta`:
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_eventual_consistency"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_eventual_consistency"
-        ```
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
+    table = db.open_table("my_table")
+    ```
 
     By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
 
-    === "Sync API"
+    ```python
+    db = lancedb.connect("./.lancedb")
+    table = db.open_table("my_table")
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_checkout_latest"
-        ```
-    === "Async API"
+    # (Other writes happen to my_table from another process)
 
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_checkout_latest"
-        ```
+    # Check for updates
+    table.checkout_latest()
+    ```
 
 === "Typescript[^1]"
 
@@ -1028,14 +957,14 @@ There are three possible settings for `read_consistency_interval`:
 
     ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
-    const tbl = await db.openTable("my_table");
+    const table = await db.openTable("my_table");
     ```
 
     For eventual consistency, specify the update interval as seconds:
 
     ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
-    const tbl = await db.openTable("my_table");
+    const table = await db.openTable("my_table");
     ```
 
 <!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,8 +1,8 @@
 ## Improving retriever performance
 
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
-VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
 
 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
 
@@ -19,7 +19,7 @@ Using different embedding models is something that's very specific to the use ca
 
 
 ## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
 
 ### Using different query types
 Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
@@ -45,14 +45,14 @@ table.add(df[["context"]].to_dict(orient="records"))
 queries = df["query"].tolist()
 ```
 
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
 
 * <b> Vector Search: </b>
 
     ```python
     table.search(quries[0], query_type="vector").limit(5).to_pandas()
     ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
 
     ```python
     table.search(quries[0]).limit(5).to_pandas()
@@ -77,7 +77,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
 * <b> Hybrid Search: </b>
 
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
     ```python
     table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
     ```
@@ -87,7 +87,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run
 
     !!! note "Note"
         By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
 
     
 

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,6 +1,6 @@
 Continuing from the previous section, we can now rerank the results using more complex rerankers.
 
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:
@@ -62,6 +62,9 @@ Let us take a look at the same datasets from the previous sections, using the sa
 | Reranked fts  | 0.672    |
 | Hybrid | 0.759 |
 
+### SQuAD Dataset
+
+
 ### Uber10K sec filing Dataset
 
 | Query Type | Hit-rate@5 |

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -1,5 +1,5 @@
 ## Finetuning the Embedding Model
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
 
@@ -16,7 +16,7 @@ validation_df.to_csv("data_val.csv", index=False)
 You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
 
 
-We parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node:
+Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
 ```python
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.readers.file import PagedCSVReader
@@ -43,7 +43,7 @@ val_dataset = generate_qa_embedding_pairs(
 )
 ```
 
-Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model:
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
 
 ```python
 from llama_index.finetuning import SentenceTransformersFinetuneEngine
@@ -57,7 +57,7 @@ finetune_engine = SentenceTransformersFinetuneEngine(
 finetune_engine.finetune()
 embed_model = finetune_engine.get_finetuned_model()
 ```
-This saves the fine tuned embedding model in `tuned_model` folder.
+This saves the fine tuned embedding model in `tuned_model` folder. This al
 
 # Evaluation results
 In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 

docs/src/hybrid_search/eval.md

@@ -3,22 +3,22 @@
 Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
 
 ## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step: reranking.
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.
 
-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example: Weighted linear combination of semantic search & keyword-based search results.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
 
-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
 
-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset or application specific so it's hard to generalize.
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
 
 ### Example evaluation of hybrid search with Reranking
 
-Here's some evaluation numbers from an experiment comparing these rerankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
 
 <b> With OpenAI ada2 embedding </b>
 
-Vector Search baseline: `0.64`
+Vector Search baseline - `0.64`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
@@ -33,7 +33,7 @@ Vector Search baseline: `0.64`
 
 <b> With OpenAI embedding-v3-small </b>
 
-Vector Search baseline: `0.59`
+Vector Search baseline - `0.59`
 
 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |

docs/src/hybrid_search/hybrid_search.md

@@ -5,46 +5,57 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
-=== "Sync API"
+```python
+import os
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search"
-    ```
-=== "Async API"
+import lancedb
+import openai
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search_async"
-    ```
+db = lancedb.connect("~/.lancedb")
 
+# Ingest embedding function in LanceDB table
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+    openai.api_key = "sk-..."
+embeddings = get_registry().get("openai").create()
+
+class Documents(LanceModel):
+    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+    text: str = embeddings.SourceField()
+
+table = db.create_table("documents", schema=Documents)
+
+data = [
+    { "text": "rebel spaceships striking from a hidden base"},
+    { "text": "have won their first victory against the evil Galactic Empire"},
+    { "text": "during the battle rebel spies managed to steal secret plans"},
+    { "text": "to the Empire's ultimate weapon the Death Star"}
+]
+
+# ingest docs with auto-vectorization
+table.add(data)
+
+# Create a fts index before the hybrid search
+table.create_fts_index("text")
+# hybrid search with default re-ranker
+results = table.search("flower moon", query_type="hybrid").to_pandas()
+```
 !!! Note
     You can also pass the vector and text query manually. This is useful if you're not using the embedding API or if you're using a separate embedder service.
 ### Explicitly passing the vector and text query
-=== "Sync API"
+```python
+vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
+text_query = "flower moon"
+results = table.search(query_type="hybrid")
+                .vector(vector_query)
+                .text(text_query)
+                .limit(5)
+                .to_pandas()
 
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text_async"
-    ```
+```
 
 By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
 
@@ -57,7 +68,7 @@ By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion scor
 
 
 ## Available Rerankers
-LanceDB provides a number of rerankers out of the box. You can use any of these rerankers by passing them to the `rerank()` method. 
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. 
 Go to [Rerankers](../reranking/index.md) to learn more about using the available rerankers and implementing custom rerankers.
 
 

docs/src/js/README.md

@@ -40,4 +40,37 @@ The [quickstart](../basic.md) contains a more complete example.
 
 ## Development
 
-See [CONTRIBUTING.md](_media/CONTRIBUTING.md) for information on how to contribute to LanceDB.
+```sh
+npm run build
+npm run test
+```
+
+### Running lint / format
+
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
+To manually lint your code you can run:
+
+```sh
+npm run lint
+```
+
+to automatically fix all fixable issues:
+
+```sh
+npm run lint-fix
+```
+
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
+
+### Generating docs
+
+```sh
+npm run docs
+
+cd ../docs
+# Asssume the virtual environment was created
+# python3 -m venv venv
+# pip install -r requirements.txt
+. ./venv/bin/activate
+mkdocs build
+```

docs/src/js/_media/CONTRIBUTING.md

@@ -1,76 +0,0 @@
-# Contributing to LanceDB Typescript
-
-This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
-
-## Project layout
-
-The Typescript package is a wrapper around the Rust library, `lancedb`. We use
-the [napi-rs](https://napi.rs/) library to create the bindings between Rust and
-Typescript.
-
-* `src/`: Rust bindings source code
-* `lancedb/`: Typescript package source code
-* `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
-
-## Development environment
-
-To set up your development environment, you will need to install the following:
-
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
-
-Initial setup:
-
-```shell
-npm install
-```
-
-### Commit Hooks
-
-It is **highly recommended** to install the [pre-commit](https://pre-commit.com/) hooks to ensure that your
-code is formatted correctly and passes basic checks before committing:
-
-```shell
-pre-commit install
-```
-
-## Development
-
-Most common development commands can be run using the npm scripts.
-
-Build the package
-
-```shell
-npm install
-npm run build
-```
-
-Lint:
-
-```shell
-npm run lint
-```
-
-Format and fix lints:
-
-```shell
-npm run lint-fix
-```
-
-Run tests:
-
-```shell
-npm test
-```
-
-To run a single test:
-
-```shell
-# Single file: table.test.ts
-npm test -- table.test.ts
-# Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
-```

docs/src/js/classes/Table.md

@@ -317,32 +317,6 @@ then call ``cleanup_files`` to remove the old files.
 
 ***
 
-### dropIndex()
-
-```ts
-abstract dropIndex(name): Promise<void>
-```
-
-Drop an index from the table.
-
-#### Parameters
-
-* **name**: `string`
-    The name of the index.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-#### Note
-
-This does not delete the index from disk, it just removes it from the table.
-To delete the index, run [Table#optimize](Table.md#optimize) after dropping the index.
-
-Use [Table.listIndices](Table.md#listindices) to find the names of the indices.
-
-***
-
 ### indexStats()
 
 ```ts
@@ -362,8 +336,6 @@ List all the stats of a specified index
 
 The stats of the index. If the index does not exist, it will return undefined
 
-Use [Table.listIndices](Table.md#listindices) to find the names of the indices.
-
 ***
 
 ### isOpen()

docs/src/js/classes/VectorQuery.md

@@ -128,24 +128,6 @@ whose data type is a fixed-size-list of floats.
 
 ***
 
-### distanceRange()
-
-```ts
-distanceRange(lowerBound?, upperBound?): VectorQuery
-```
-
-#### Parameters
-
-* **lowerBound?**: `number`
-
-* **upperBound?**: `number`
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-***
-
 ### distanceType()
 
 ```ts
@@ -546,22 +528,6 @@ distance between the query vector and the actual uncompressed vector.
 
 ***
 
-### rerank()
-
-```ts
-rerank(reranker): VectorQuery
-```
-
-#### Parameters
-
-* **reranker**: [`Reranker`](../namespaces/rerankers/interfaces/Reranker.md)
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-***
-
 ### select()
 
 ```ts

docs/src/js/globals.md

@@ -7,7 +7,6 @@
 ## Namespaces
 
 - [embedding](namespaces/embedding/README.md)
-- [rerankers](namespaces/rerankers/README.md)
 
 ## Enumerations
 

docs/src/js/interfaces/IvfPqOptions.md

@@ -68,21 +68,6 @@ The default value is 50.
 
 ***
 
-### numBits?
-
-```ts
-optional numBits: number;
-```
-
-Number of bits per sub-vector.
-
-This value controls how much each subvector is compressed.  The more bits the more
-accurate the index will be but the slower search.  The default is 8 bits.
-
-The number of bits must be 4 or 8.
-
-***
-
 ### numPartitions?
 
 ```ts

docs/src/js/namespaces/rerankers/README.md

@@ -1,17 +0,0 @@
-[**@lancedb/lancedb**](../../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../../globals.md) / rerankers
-
-# rerankers
-
-## Index
-
-### Classes
-
-- [RRFReranker](classes/RRFReranker.md)
-
-### Interfaces
-
-- [Reranker](interfaces/Reranker.md)

docs/src/js/namespaces/rerankers/classes/RRFReranker.md

@@ -1,66 +0,0 @@
-[**@lancedb/lancedb**](../../../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../../../globals.md) / [rerankers](../README.md) / RRFReranker
-
-# Class: RRFReranker
-
-Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
-
-Internally this uses the Rust implementation
-
-## Constructors
-
-### new RRFReranker()
-
-```ts
-new RRFReranker(inner): RRFReranker
-```
-
-#### Parameters
-
-* **inner**: `RrfReranker`
-
-#### Returns
-
-[`RRFReranker`](RRFReranker.md)
-
-## Methods
-
-### rerankHybrid()
-
-```ts
-rerankHybrid(
-   query,
-   vecResults,
-   ftsResults): Promise<RecordBatch<any>>
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **vecResults**: `RecordBatch`&lt;`any`&gt;
-
-* **ftsResults**: `RecordBatch`&lt;`any`&gt;
-
-#### Returns
-
-`Promise`&lt;`RecordBatch`&lt;`any`&gt;&gt;
-
-***
-
-### create()
-
-```ts
-static create(k): Promise<RRFReranker>
-```
-
-#### Parameters
-
-* **k**: `number` = `60`
-
-#### Returns
-
-`Promise`&lt;[`RRFReranker`](RRFReranker.md)&gt;

docs/src/js/namespaces/rerankers/interfaces/Reranker.md

@@ -1,30 +0,0 @@
-[**@lancedb/lancedb**](../../../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../../../globals.md) / [rerankers](../README.md) / Reranker
-
-# Interface: Reranker
-
-## Methods
-
-### rerankHybrid()
-
-```ts
-rerankHybrid(
-   query,
-   vecResults,
-   ftsResults): Promise<RecordBatch<any>>
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **vecResults**: `RecordBatch`&lt;`any`&gt;
-
-* **ftsResults**: `RecordBatch`&lt;`any`&gt;
-
-#### Returns
-
-`Promise`&lt;`RecordBatch`&lt;`any`&gt;&gt;

docs/src/migration.md

@@ -66,7 +66,7 @@ the size of the data.
 
 ### Embedding Functions
 
-The embedding API has been completely reworked, and it now more closely resembles the Python API, including the new [embedding registry](./js/classes/embedding.EmbeddingFunctionRegistry.md):
+The embedding API has been completely reworked, and it now more closely resembles the Python API, including the new [embedding registry](./js/classes/embedding.EmbeddingFunctionRegistry.md)
 
 === "vectordb (deprecated)"
 

docs/src/notebooks/embedding_tuner.ipynb

@@ -207,7 +207,7 @@
       "cell_type": "markdown",
       "source": [
         "## The dataset\n",
-        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retriever.\n",
+        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retreiver.\n",
         "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo\n"
       ],
       "metadata": {

docs/src/notebooks/hybrid_search.ipynb

@@ -477,7 +477,7 @@
    "source": [
     "## Vector Search\n",
     "\n",
-    "Average latency: `3.48 ms ± 71.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
+    "avg latency - `3.48 ms ± 71.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
    ]
   },
   {
@@ -597,7 +597,7 @@
     "`LinearCombinationReranker(weight=0.7)` is used as the default reranker for reranking the hybrid search results if the reranker isn't specified explicitly.\n",
     "The `weight` param controls the weightage provided to vector search score. The weight of `1-weight` is applied to FTS scores when reranking.\n",
     "\n",
-    "Latency: `71 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
+    "Latency - `71 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)`"
    ]
   },
   {
@@ -675,9 +675,9 @@
    },
    "source": [
     "### Cohere Reranker\n",
-    "This uses Cohere's Reranking API to re-rank  the results. It accepts the reranking model name as a parameter. By default it uses the english-v3 model but you can easily switch to a multi-lingual model.\n",
+    "This uses Cohere's Reranking API to re-rank  the results. It accepts the reranking model name as a parameter. By Default it uses the english-v3 model but you can easily switch to a multi-lingual model.\n",
     "\n",
-    "Latency: `605 ms ± 78.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
+    "latency - `605 ms ± 78.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
    ]
   },
   {
@@ -1165,7 +1165,7 @@
    },
    "source": [
     "### ColBERT Reranker\n",
-    "Colbert Reranker is powered by ColBERT model. It runs locally using the huggingface implementation.\n",
+    "Colber Reranker is powered by ColBERT model. It runs locally using the huggingface implementation.\n",
     "\n",
     "Latency - `950 ms ± 5.78 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`\n",
     "\n",
@@ -1489,9 +1489,9 @@
    },
    "source": [
     "### Cross Encoder Reranker\n",
-    "Uses cross encoder models are rerankers. Uses sentence transformer implementation locally\n",
+    "Uses cross encoder models are rerankers. Uses sentence transformer implemntation locally\n",
     "\n",
-    "Latency: `1.38 s ± 64.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
+    "Latency - `1.38 s ± 64.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)`"
    ]
   },
   {
@@ -1771,10 +1771,10 @@
    "source": [
     "### (Experimental) OpenAI Reranker\n",
     "\n",
-    "This prompts a chat model to rerank results and is not a dedicated reranker model. This should be treated as experimental. You might exceed the token limit so set the search limits based on your token limit.\n",
-    "NOTE: It is recommended to use `gpt-4-turbo-preview` as older models might lead to bad behaviour\n",
+    "This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental. You might run out of token limit so set the search limits based on your token limit.\n",
+    "NOTE: It is recommended to use `gpt-4-turbo-preview`, older models might lead to bad behaviour\n",
     "\n",
-    "Latency: `Can take 10s of seconds if using GPT-4 model`"
+    "Latency - `Can take 10s of seconds if using GPT-4 model`"
    ]
   },
   {
@@ -1817,7 +1817,7 @@
    },
    "source": [
     "## Use your custom Reranker\n",
-    "Hybrid search in LanceDB is designed to be very flexible. You can easily plug in your own Re-reranking logic. To do so, you simply need to implement the base Reranker class:"
+    "Hybrid search in LanceDB is designed to be very flexible. You can easily plug in your own Re-reranking logic. To do so, you simply need to implement the base Reranker class"
    ]
   },
   {
@@ -1849,9 +1849,9 @@
    "source": [
     "### Custom Reranker based on CohereReranker\n",
     "\n",
-    "For the sake of simplicity let's build a custom reranker that enhances the Cohere Reranker by accepting a filter query, and accepts other CohereReranker params as kwargs.\n",
+    "For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.\n",
     "\n",
-    "For this toy example let's say we want to get rid of docs that represent a table of contents or appendix, as these are semantically close to representing costs but don't represent the specific reasons why operating costs were high."
+    "For this toy example let's say we want to get rid of docs that represent a table of contents, appendix etc. as these are semantically close of representing costs but this isn't something we are interested in because they don't represent the specific reasons why operating costs were high. They simply represent the costs."
    ]
   },
   {
@@ -1969,7 +1969,7 @@
     "id": "b3b5464a-7252-4eab-aaac-9b0eae37496f"
    },
    "source": [
-    "As you can see, the document containing the table of contents no longer shows up."
+    "As you can see the document containing the Table of contetnts of spending no longer shows up"
    ]
   }
  ],

docs/src/notebooks/lancedb_reranking.ipynb

@@ -49,7 +49,7 @@
       },
       "source": [
         "## What is a retriever\n",
-        "VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.\n",
+        "VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.\n",
         "\n",
         "<img src=\"https://llmstack.ai/assets/images/rag-f517f1f834bdbb94a87765e0edd40ff2.png\" />\n",
         "\n",
@@ -64,7 +64,7 @@
         "- Fine-tuning the embedding models\n",
         "- Using different embedding models\n",
         "\n",
-        "Obviously, the above list is not exhaustive. There are other subtler ways that can improve retrieval performance like alternative chunking algorithms, using different distance/similarity metrics, and more. For brevity, we'll only cover high level and more impactful techniques here.\n",
+        "Obviously, the above list is not exhaustive. There are other subtler ways that can improve retrieval performance like experimenting chunking algorithms, using different distance/similarity metrics etc. But for brevity, we'll only cover high level and more impactful techniques here.\n",
         "\n"
       ]
     },
@@ -77,7 +77,7 @@
         "# LanceDB\n",
         "- Multimodal DB for AI\n",
         "- Powered by an innovative & open-source in-house file format\n",
-        "- Zero setup\n",
+        "- 0 Setup\n",
         "- Scales up on disk storage\n",
         "- Native support for vector, full-text(BM25) and hybrid search\n",
         "\n",
@@ -92,8 +92,8 @@
       },
       "source": [
         "## The dataset\n",
-        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retriever.\n",
-        "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo.\n"
+        "The dataset we'll use is a synthetic QA dataset generated from LLama2 review paper. The paper was divided into chunks, with each chunk being a unique context. An LLM was prompted to ask questions relevant to the context for testing a retreiver.\n",
+        "The exact code and other utility functions for this can be found in [this](https://github.com/lancedb/ragged) repo\n"
       ]
     },
     {
@@ -594,10 +594,10 @@
       },
       "source": [
         "## Ingestion\n",
-        "Let us now ingest the contexts in LanceDB. The steps will be:\n",
+        "Let us now ingest the contexts in LanceDB\n",
         "\n",
         "- Create a schema (Pydantic or Pyarrow)\n",
-        "- Select an embedding model from LanceDB Embedding API (to allow automatic vectorization of data)\n",
+        "- Select an embedding model from LanceDB Embedding API (Allows automatic vectorization of data)\n",
         "- Ingest the contexts\n"
       ]
     },
@@ -841,7 +841,7 @@
       },
       "source": [
         "## Different Query types in LanceDB\n",
-        "LanceDB allows switching query types with by setting `query_type` argument, which defaults to `vector` when using Embedding API. In this example we'll use `JinaReranker` which is one of many rerankers supported by LanceDB.\n",
+        "LanceDB allows switching query types with by setting `query_type` argument, which defaults to `vector` when using Embedding API. In this example we'll use `JinaReranker` which is one of many rerankers supported by LanceDB\n",
         "\n",
         "### Vector search:\n",
         "Vector search\n",
@@ -1446,11 +1446,11 @@
       "source": [
         "## Takeaways & Tradeoffs\n",
         "\n",
-        "* **Rerankers significantly improve accuracy at little cost.** Using Hybrid search and/or rerankers can significantly improve retrieval performance without spending any additional time or effort on tuning embedding models, generators, or dissecting the dataset.\n",
+        "* **Easiest method to significantly improve accuracy** Using Hybrid search and/or rerankers can significantly improve retrieval performance without spending any additional time or effort on tuning embedding models, generators, or dissecting the dataset.\n",
         "\n",
         "* **Reranking is an expensive operation.** Depending on the type of reranker you choose, they can incur significant latecy to query times. Although some API-based rerankers can be significantly faster.\n",
         "\n",
-        "* **Pre-warmed GPU environments reduce latency.**  When using models locally, having a warmed-up GPU environment will significantly reduce latency. This is especially useful if the application doesn't need to be strictly realtime. Pre-warming comes at the expense of GPU resources."
+        "*  When using models locally, having a warmed-up GPU environment will significantly reduce latency. This is specially useful if the application doesn't need to be strcitly realtime. The tradeoff being GPU resources."
       ]
     },
     {
@@ -1504,4 +1504,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
+}

docs/src/notebooks/tables_guide.ipynb

@@ -114,17 +114,14 @@
     }
    ],
    "source": [
-    "import pandas as pd\n",
+    "data = [\n",
+    "    {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
+    "    {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n",
+    "]\n",
     "\n",
-    "data = pd.DataFrame(\n",
-    "    {\n",
-    "        \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n",
-    "        \"lat\": [45.5, 40.1],\n",
-    "        \"long\": [-122.7, -74.1],\n",
-    "    }\n",
-    ")\n",
-    "db.create_table(\"my_table_pandas\", data)\n",
-    "db[\"my_table_pandas\"].head()"
+    "db.create_table(\"table2\", data)\n",
+    "\n",
+    "db[\"table2\"].head() "
    ]
   },
   {
@@ -167,7 +164,7 @@
     "import pyarrow as pa\n",
     "\n",
     "custom_schema = pa.schema([\n",
-    "pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n",
+    "pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
     "pa.field(\"lat\", pa.float32()),\n",
     "pa.field(\"long\", pa.float32())\n",
     "])\n",

docs/src/python/pandas_and_pyarrow.md

@@ -8,55 +8,54 @@ and PyArrow. The sequence of steps in a typical workflow is shown below.
 
 First, we need to connect to a LanceDB database.
 
-=== "Sync API"
+```py
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb"
-    ```
-=== "Async API"
+import lancedb
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb_async"
-    ```
+db = lancedb.connect("data/sample-lancedb")
+```
 
 We can load a Pandas `DataFrame` to LanceDB directly.
 
-=== "Sync API"
+```py
+import pandas as pd
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_python.py:create_table_pandas"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_python.py:create_table_pandas_async"
-    ```
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+```
 
 Similar to the [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html) method, LanceDB's
 [`db.create_table()`](python.md/#lancedb.db.DBConnection.create_table) accepts data in a variety of forms.
 
 If you have a dataset that is larger than memory, you can create a table with `Iterator[pyarrow.RecordBatch]` to lazily load the data:
 
-=== "Sync API"
+```py
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-iterable"
-    --8<-- "python/python/tests/docs/test_python.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_python.py:make_batches"
-    --8<-- "python/python/tests/docs/test_python.py:create_table_iterable"
-    ```
-=== "Async API"
+from typing import Iterable
+import pyarrow as pa
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:import-iterable"
-    --8<-- "python/python/tests/docs/test_python.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_python.py:make_batches"
-    --8<-- "python/python/tests/docs/test_python.py:create_table_iterable_async"
-    ```
+def make_batches() -> Iterable[pa.RecordBatch]:
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"])
+
+schema=pa.schema([
+    pa.field("vector", pa.list_(pa.float32())),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+table = db.create_table("iterable_table", data=make_batches(), schema=schema)
+```
 
 You will find detailed instructions of creating a LanceDB dataset in
 [Getting Started](../basic.md#quick-start) and [API](python.md/#lancedb.db.DBConnection.create_table)
@@ -66,16 +65,15 @@ sections.
 
 We can now perform similarity search via the LanceDB Python API.
 
-=== "Sync API"
+```py
+# Open the table previously created.
+table = db.open_table("pd_table")
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:vector_search"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:vector_search_async"
-    ```
+query_vector = [100, 100]
+# Pandas DataFrame
+df = table.search(query_vector).limit(1).to_pandas()
+print(df)
+```
 
 ```
     vector     item  price    _distance
@@ -85,13 +83,16 @@ We can now perform similarity search via the LanceDB Python API.
 If you have a simple filter, it's faster to provide a `where` clause to LanceDB's `search` method.
 For more complex filters or aggregations, you can always resort to using the underlying `DataFrame` methods after performing a search.
 
-=== "Sync API"
+```python
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:vector_search_with_filter"
-    ```
-=== "Async API"
+# Apply the filter via LanceDB
+results = table.search([100, 100]).where("price < 15").to_pandas()
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
 
-    ```python
-    --8<-- "python/python/tests/docs/test_python.py:vector_search_with_filter_async"
-    ```
+# Apply the filter via Pandas
+df = results = table.search([100, 100]).to_pandas()
+results = df[df.price < 15]
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+```

docs/src/python/polars_arrow.md

@@ -2,29 +2,38 @@
 
 LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow under the hood. A deeper integration between Lance Tables and Polars DataFrames is in progress, but at the moment, you can read a Polars DataFrame into LanceDB and output the search results from a query to a Polars DataFrame.
 
-
 ## Create & Query LanceDB Table
 
 ### From Polars DataFrame
 
 First, we connect to a LanceDB database.
 
-
 ```py
---8<-- "python/python/tests/docs/test_python.py:import-lancedb"
---8<-- "python/python/tests/docs/test_python.py:connect_to_lancedb"
+import lancedb
+
+db = lancedb.connect("data/polars-lancedb")
 ```
 
 We can load a Polars `DataFrame` to LanceDB directly.
 
 ```py
---8<-- "python/python/tests/docs/test_python.py:import-polars"
---8<-- "python/python/tests/docs/test_python.py:create_table_polars"
+import polars as pl
+
+data = pl.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pl_table", data=data)
 ```
+
 We can now perform similarity search via the LanceDB Python API.
 
 ```py
---8<-- "python/python/tests/docs/test_python.py:vector_search_polars"
+query = [3.0, 4.0]
+result = table.search(query).limit(1).to_polars()
+print(result)
+print(type(result))
 ```
 
 In addition to the selected columns, LanceDB also returns a vector
@@ -50,16 +59,33 @@ Note that the type of the result from a table search is a Polars DataFrame.
 Alternately, we can create an empty LanceDB Table using a Pydantic schema and populate it with a Polars DataFrame.
 
 ```py
---8<-- "python/python/tests/docs/test_python.py:import-polars"
---8<-- "python/python/tests/docs/test_python.py:import-lancedb-pydantic"
---8<-- "python/python/tests/docs/test_python.py:class_Item"
---8<-- "python/python/tests/docs/test_python.py:create_table_pydantic"
+import polars as pl
+from lancedb.pydantic import Vector, LanceModel
+
+
+class Item(LanceModel):
+    vector: Vector(2)
+    item: str
+    price: float
+
+data = {
+    "vector": [[3.1, 4.1]],
+    "item": "foo",
+    "price": 10.0,
+}
+
+table = db.create_table("test_table", schema=Item)
+df = pl.DataFrame(data)
+# Add Polars DataFrame to table
+table.add(df)
 ```
 
 The table can now be queried as usual.
 
 ```py
---8<-- "python/python/tests/docs/test_python.py:vector_search_polars"
+result = table.search([3.0, 4.0]).limit(1).to_polars()
+print(result)
+print(type(result))
 ```
 
 ```
@@ -82,7 +108,8 @@ As you iterate on your application, you'll likely need to work with the whole ta
 LanceDB tables can also be converted directly into a polars LazyFrame for further processing.
 
 ```python
---8<-- "python/python/tests/docs/test_python.py:dump_table_lazyform"
+ldf = table.to_polars()
+print(type(ldf))
 ```
 
 Unlike the search result from a query, we can see that the type of the result is a LazyFrame.
@@ -94,7 +121,7 @@ Unlike the search result from a query, we can see that the type of the result is
 We can now work with the LazyFrame as we would in Polars, and collect the first result.
 
 ```python
---8<-- "python/python/tests/docs/test_python.py:print_table_lazyform"
+print(ldf.first().collect())
 ```
 
 ```

docs/src/python/python.md

@@ -133,10 +133,6 @@ lists the indices that LanceDb supports.
 
 ::: lancedb.index.IvfPq
 
-::: lancedb.index.HnswPq
-
-::: lancedb.index.HnswSq
-
 ::: lancedb.index.IvfFlat
 
 ## Querying (Asynchronous)
@@ -147,19 +143,8 @@ to return the entire (typically filtered) table. Vector searches return the
 rows nearest to a query vector and can be created with the
 [AsyncTable.vector_search][lancedb.table.AsyncTable.vector_search] method.
 
+::: lancedb.query.AsyncQueryBase
 
 ::: lancedb.query.AsyncQuery
-    options:
-      inherited_members: true
 
 ::: lancedb.query.AsyncVectorQuery
-    options:
-      inherited_members: true
-
-::: lancedb.query.AsyncFTSQuery
-    options:
-      inherited_members: true
-
-::: lancedb.query.AsyncHybridQuery
-    options:
-      inherited_members: true

docs/src/rag/adaptive_rag.md

@@ -2,7 +2,7 @@
 ====================================================================
 Adaptive RAG introduces a RAG technique that combines query analysis with self-corrective RAG. 
 
-For Query Analysis, it uses a small classifier(LLM), to decide the query’s complexity. Query Analysis guides adjustment between different retrieval strategies: No retrieval, Single-shot RAG or Iterative RAG.
+For Query Analysis, it uses a small classifier(LLM), to decide the query’s complexity. Query Analysis helps routing smoothly to adjust between different retrieval strategies No retrieval, Single-shot RAG or Iterative RAG.
 
 **[Official Paper](https://arxiv.org/pdf/2403.14403)**
 
@@ -12,9 +12,9 @@ For Query Analysis, it uses a small classifier(LLM), to decide the query’s com
   </figcaption>
 </figure>
 
-**[Official Implementation](https://github.com/starsuzi/Adaptive-RAG)**
+**[Offical Implementation](https://github.com/starsuzi/Adaptive-RAG)**
 
-Here’s a code snippet for query analysis:
+Here’s a code snippet for query analysis
 
 ```python
 from langchain_core.prompts import ChatPromptTemplate
@@ -35,7 +35,7 @@ llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
 structured_llm_router = llm.with_structured_output(RouteQuery)
 ```
 
-The following example defines and queries a retriever:
+For defining and querying retriever
 
 ```python
 # add documents in LanceDB
@@ -48,4 +48,4 @@ retriever = vectorstore.as_retriever()
 # query using defined retriever
 question = "How adaptive RAG works"
 docs = retriever.get_relevant_documents(question)
-```
+```

docs/src/rag/advanced_techniques/flare.md

@@ -11,7 +11,7 @@ FLARE, stands for Forward-Looking Active REtrieval augmented generation is a gen
 
 [![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)
 
-Here’s a code snippet for using FLARE with Langchain:
+Here’s a code snippet for using FLARE with Langchain
 
 ```python
 from langchain.vectorstores import LanceDB
@@ -35,4 +35,4 @@ flare = FlareChain.from_llm(llm=llm,retriever=vector_store_retriever,max_generat
 result = flare.run(input_text)
 ```
 
-[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)
+[![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb)

docs/src/rag/advanced_techniques/hyde.md

@@ -11,7 +11,7 @@ HyDE, stands for Hypothetical Document Embeddings is an approach used for precis
 
 [![Open In Colab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb)
 
-Here’s a code snippet for using HyDE with Langchain:
+Here’s a code snippet for using HyDE with Langchain
 
 ```python
 from langchain.llms import OpenAI

docs/src/rag/agentic_rag.md

@@ -1,6 +1,6 @@
 **Agentic RAG 🤖**
 ====================================================================
-Agentic RAG introduces an advanced framework for answering questions by using intelligent agents instead of just relying on large language models. These agents act like expert researchers, handling complex tasks such as detailed planning, multi-step reasoning, and using external tools. They navigate multiple documents, compare information, and generate accurate answers. This system is easily scalable, with each new document set managed by a sub-agent, making it a powerful tool for tackling a wide range of information needs.
+Agentic RAG is Agent-based RAG introduces an advanced framework for answering questions by using intelligent agents instead of just relying on large language models. These agents act like expert researchers, handling complex tasks such as detailed planning, multi-step reasoning, and using external tools. They navigate multiple documents, compare information, and generate accurate answers. This system is easily scalable, with each new document set managed by a sub-agent, making it a powerful tool for tackling a wide range of information needs.
 
 <figure markdown="span">
   ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/agentic_rag.png)
@@ -9,7 +9,7 @@ Agentic RAG introduces an advanced framework for answering questions by using in
 
 [![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)
 
-Here’s a code snippet for defining retriever using Langchain:
+Here’s a code snippet for defining retriever using Langchain
 
 ```python
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -41,7 +41,7 @@ retriever = vectorstore.as_retriever()
 
 ```
 
-Here is an agent that formulates an improved query for better retrieval results and then grades the retrieved documents:
+Agent that formulates an improved query for better retrieval results and then grades the retrieved documents
 
 ```python
 def grade_documents(state) -> Literal["generate", "rewrite"]:
@@ -98,4 +98,4 @@ def rewrite(state):
     return {"messages": [response]}
 ```
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb)

docs/src/rag/corrective_rag.md

@@ -4,7 +4,7 @@
 Corrective-RAG (CRAG) is a strategy for Retrieval-Augmented Generation (RAG) that includes self-reflection and self-grading of retrieved documents. Here’s a simplified breakdown of the steps involved:
 
 1. **Relevance Check**: If at least one document meets the relevance threshold, the process moves forward to the generation phase.
-2. **Knowledge Refinement**: Before generating an answer, the process refines the knowledge by dividing the document into smaller segments called "knowledge strips".
+2. **Knowledge Refinement**: Before generating an answer, the process refines the knowledge by dividing the document into smaller segments called "knowledge strips."
 3. **Grading and Filtering**: Each "knowledge strip" is graded, and irrelevant ones are filtered out.
 4. **Additional Data Source**: If all documents are below the relevance threshold, or if the system is unsure about their relevance, it will seek additional information by performing a web search to supplement the retrieved data.
 
@@ -19,11 +19,11 @@ Above steps are mentioned in
 
 Corrective Retrieval-Augmented Generation (CRAG) is a method that works like a **built-in fact-checker**.
 
-**[Official Implementation](https://github.com/HuskyInSalt/CRAG)**
+**[Offical Implementation](https://github.com/HuskyInSalt/CRAG)**
 
 [![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)
 
-Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), and retrieves the relevant documents:
+Here’s a code snippet for defining a table with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/), and retrieves the relevant documents.
 
 ```python
 import pandas as pd
@@ -115,6 +115,6 @@ def grade_documents(state):
     }
 ```
 
-Check Colab for the Implementation of CRAG with Langgraph:
+Check Colab for the Implementation of CRAG with Langgraph 
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb)

docs/src/rag/graph_rag.md

@@ -6,7 +6,7 @@ One of the main benefits of Graph RAG is its ability to capture and represent co
 
 **[Official Paper](https://arxiv.org/pdf/2404.16130)**
 
-**[Official Implementation](https://github.com/microsoft/graphrag)**
+**[Offical Implementation](https://github.com/microsoft/graphrag)**
 
 [Microsoft Research Blog](https://www.microsoft.com/en-us/research/blog/graphrag-unlocking-llm-discovery-on-narrative-private-data/)
 
@@ -39,16 +39,16 @@ python3 -m graphrag.index --root dataset-dir
 
 - **Execute Query**
 
-Global Query Execution gives a broad overview of dataset:
+Global Query Execution gives a broad overview of dataset
 
 ```bash
 python3 -m graphrag.query --root dataset-dir --method global "query-question"
 ```
 
-Local Query Execution gives a detailed and specific answers based on the context of the entities:
+Local Query Execution gives a detailed and specific answers based on the context of the entities
 
 ```bash
 python3 -m graphrag.query --root  dataset-dir --method local "query-question"
 ```
 
-[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Graphrag/main.ipynb)
+[![Open In Colab](../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Graphrag/main.ipynb)

docs/src/rag/multi_head_rag.md

@@ -15,7 +15,7 @@ MRAG is cost-effective and energy-efficient because it avoids extra LLM queries,
 
 **[Official Implementation](https://github.com/spcl/MRAG)**
 
-Here’s a code snippet for defining different embedding spaces with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/):
+Here’s a code snippet for defining different embedding spaces with the [Embedding API](https://lancedb.github.io/lancedb/embeddings/embedding_functions/)
 
 ```python
 import lancedb
@@ -44,6 +44,6 @@ class Space3(LanceModel):
     vector: Vector(model3.ndims()) = model3.VectorField()
 ```
 
-Create different tables using defined embedding spaces, then make queries to each embedding space. Use the resulting closest documents from each embedding space to generate answers.
+Create different tables using defined embedding spaces, then make queries to each embedding space. Use the resulted closest documents from each embedding space to generate answers.
 
 

docs/src/rag/self_rag.md

@@ -1,6 +1,6 @@
 **Self RAG 🤳**
 ====================================================================
-Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better retrieved information, generated text, and validation, without loss of flexibility. Unlike the traditional Retrieval-Augmented Generation (RAG) method, Self-RAG retrieves information as needed, can skip retrieval if not needed, and evaluates its own output while generating text. It also uses a process to pick the best output based on different preferences.
+Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better retrieved information, generated text, and  checking their own work, all without losing their flexibility. Unlike the traditional Retrieval-Augmented Generation (RAG) method, Self-RAG retrieves information as needed, can skip retrieval if not needed, and evaluates its own output while generating text. It also uses a process to pick the best output based on different preferences.
 
 **[Official Paper](https://arxiv.org/pdf/2310.11511)**
 
@@ -10,11 +10,11 @@ Self-RAG is a strategy for Retrieval-Augmented Generation (RAG) to get better re
   </figcaption>
 </figure>
 
-**[Official Implementation](https://github.com/AkariAsai/self-rag)**
+**[Offical Implementation](https://github.com/AkariAsai/self-rag)**
 
 Self-RAG starts by generating a response without retrieving extra info if it's not needed. For questions that need more details, it retrieves to get the necessary information.
 
-Here’s a code snippet for defining retriever using Langchain:
+Here’s a code snippet for defining retriever using Langchain
 
 ```python
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -46,7 +46,7 @@ retriever = vectorstore.as_retriever()
 
 ```
 
-The following functions grade the retrieved documents and formulate an improved query for better retrieval results, if required:
+Functions that grades the retrieved documents and if required formulates an improved query for better retrieval results  
 
 ```python
 def grade_documents(state) -> Literal["generate", "rewrite"]:
@@ -93,4 +93,4 @@ def rewrite(state):
     model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
     response = model.invoke(msg)
     return {"messages": [response]}
-```
+```

docs/src/rag/sfr_rag.md

@@ -1,8 +1,8 @@
 **SFR RAG 📑**
 ====================================================================
-Salesforce AI Research introduced SFR-RAG, a 9-billion-parameter language model trained with a significant emphasis on reliable, precise, and faithful contextual generation abilities specific to real-world RAG use cases and relevant agentic tasks. It targets precise factual knowledge extraction, distinction between relevant and distracting contexts, citation of appropriate sources along with answers, production of complex and multi-hop reasoning over multiple contexts, consistent format following, as well as minimization of hallucination over unanswerable queries.
+Salesforce AI Research introduces SFR-RAG, a 9-billion-parameter language model trained with a significant emphasis on reliable, precise, and faithful contextual generation abilities specific to real-world RAG use cases and relevant agentic tasks. They include precise factual knowledge extraction, distinguishing relevant against distracting contexts, citing appropriate sources along with answers, producing complex and multi-hop reasoning over multiple contexts, consistent format following, as well as refraining from hallucination over unanswerable queries.
 
-**[Official Implementation](https://github.com/SalesforceAIResearch/SFR-RAG)**
+**[Offical Implementation](https://github.com/SalesforceAIResearch/SFR-RAG)**
 
 <figure markdown="span">
   ![agent-based-rag](https://raw.githubusercontent.com/lancedb/assets/main/docs/assets/rag/salesforce_contextbench.png)

docs/src/reranking/answerdotai.md

@@ -1,6 +1,7 @@
 # AnswersDotAI Rerankers
 
-This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/rerankers) to rerank the search results, providing a lightweight, low-dependency, unified API to use all common reranking and cross-encoder models.
+This integration allows using answersdotai's rerankers to rerank the search results. [Rerankers](https://github.com/AnswerDotAI/rerankers)
+A lightweight, low-dependency, unified API to use all common reranking and cross-encoder models.
 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
@@ -44,10 +45,10 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here: https://github.com/AnswerDotAI/rerankers. |
+| `model_type` | `str` | `"colbert"` | The type of model to use. Supported model types can be found here - https://github.com/AnswerDotAI/rerankers |
 | `model_name` | `str` | `"answerdotai/answerai-colbert-small-v1"` | The name of the reranker model to use. |
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
 
 
 
@@ -57,17 +58,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/cohere.md

@@ -1,6 +1,6 @@
 # Cohere Reranker
 
-This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this reranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
+This re-ranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
 
 
 !!! note
@@ -62,17 +62,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/colbert.md

@@ -1,6 +1,6 @@
 # ColBERT Reranker
 
-This reranker uses ColBERT model to rerank the search results. You can use this reranker by passing `ColbertReranker()` to the `rerank()` method. 
+This re-ranker uses ColBERT model to rerank the search results. You can use this re-ranker by passing `ColbertReranker()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
@@ -46,7 +46,7 @@ Accepted Arguments
 | `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
 
 
 ## Supported Scores for each query type
@@ -55,17 +55,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/cross_encoder.md

@@ -1,6 +1,6 @@
 # Cross Encoder Reranker
 
-This reranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this reranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
+This re-ranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this re-ranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
@@ -46,7 +46,7 @@ Accepted Arguments
 | `model_name` | `str` | `""cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
 
 ## Supported Scores for each query type
 You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
@@ -54,17 +54,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/custom_reranker.md

@@ -1,10 +1,9 @@
 ## Building Custom Rerankers
 You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Optionally, you can also implement the `rerank_vector()` and `rerank_fts()` methods if you want to support reranking for vector and FTS search separately.
+Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
 
 The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
 
-Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores:
-
 ```python
 
 from lancedb.rerankers import Reranker
@@ -43,7 +42,7 @@ class MyReranker(Reranker):
 ```
 
 ### Example of a Custom Reranker
-For the sake of simplicity let's build custom reranker that enhances the Cohere Reranker by accepting a filter query, and accepts other CohereReranker params as kwargs.
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
 
 ```python
 
@@ -84,6 +83,6 @@ class ModifiedCohereReranker(CohereReranker):
 ```
 
 !!! tip
-    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be converted to other data types like pandas dataframe, pydict, pylist etc.
+    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be convered to other data types like pandas dataframe, pydict, pylist etc.
 
-    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.
+    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/reranking/index.md

@@ -13,7 +13,7 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 
 
 ## Using a Reranker
-Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder:
+Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
 
 ```python
 import lancedb
@@ -36,14 +36,14 @@ tbl = db.create_table("test", data)
 reranker = CohereReranker(api_key="your_api_key")
 
 # Run vector search with a reranker
-result = tbl.search("hello").rerank(reranker).to_list() 
+result = tbl.query("hello").rerank(reranker).to_list() 
 
 # Run FTS search with a reranker
-result = tbl.search("hello", query_type="fts").rerank(reranker).to_list()
+result = tbl.query("hello", query_type="fts").rerank(reranker).to_list()
 
 # Run hybrid search with a reranker
 tbl.create_fts_index("text")
-result = tbl.search("hello", query_type="hybrid").rerank(reranker).to_list()
+result = tbl.query("hello", query_type="hybrid").rerank(reranker).to_list()
 ```
 
 ### Multi-vector reranking
@@ -64,7 +64,7 @@ reranked = reranker.rerank_multivector([res1, res2, res3],  deduplicate=True)
 ```
     
 ## Available Rerankers
-LanceDB comes with the following built-in rerankers:
+LanceDB comes with some built-in rerankers. Here are some of the rerankers that are available in LanceDB:
 
 - [Cohere Reranker](./cohere.md)
 - [Cross Encoder Reranker](./cross_encoder.md)
@@ -78,4 +78,4 @@ LanceDB comes with the following built-in rerankers:
 
 ## Creating Custom Rerankers
 
-LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.
+LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.

docs/src/reranking/jina.md

@@ -1,6 +1,6 @@
 # Jina Reranker
 
-This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this reranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
+This re-ranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this re-ranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
 
 
 !!! note
@@ -48,11 +48,11 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker. |
+| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker/|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
 | `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
 | `api_key` | `str` | `None` | The API key for the Jina API. If not provided, the `JINA_API_KEY` environment variable is used. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
 
 
 
@@ -62,17 +62,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/linear_combination.md

@@ -1,9 +1,9 @@
 # Linear Combination Reranker
 
 !!! note
-    This is deprecated. It is recommended to use the `RRFReranker` instead, if you want to use a score-based reranker.
+    This is depricated. It is recommended to use the `RRFReranker` instead, if you want to use a score based reranker.
 
-The Linear Combination Reranker combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified, and defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
 
 !!! note
     Supported Query Types: Hybrid
@@ -51,5 +51,5 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column |
-| `all` | ✅ Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |

docs/src/reranking/openai.md

@@ -1,11 +1,11 @@
 # OpenAI Reranker (Experimental)
 
-This reranker uses OpenAI chat model to rerank the search results. You can use this reranker by passing `OpenAI()` to the `rerank()` method. 
+This re-ranker uses OpenAI chat model to rerank the search results. You can use this re-ranker by passing `OpenAI()` to the `rerank()` method. 
 !!! note
     Supported Query Types: Hybrid, Vector, FTS
 
 !!! warning
-    This reranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
+    This re-ranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
 
 ```python
 import numpy
@@ -47,7 +47,7 @@ Accepted Arguments
 | --- | --- | --- | --- |
 | `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.|
 | `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
 | `api_key` | str | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable.
 
 
@@ -57,17 +57,17 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ❌ Not Supported | Results have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
 
 ### Vector Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have vector(`_distance`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
 
 ### FTS Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Results only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Results have FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/rrf.md

@@ -1,6 +1,6 @@
 # Reciprocal Rank Fusion Reranker
 
-This is the default reranker used by LanceDB hybrid search. Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
+This is the default re-ranker used by LanceDB hybrid search. Reciprocal Rank Fusion (RRF) is an algorithm that evaluates the search scores by leveraging the positions/rank of the documents. The implementation follows this [paper](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf).
 
 
 !!! note
@@ -39,7 +39,7 @@ Accepted Arguments
 ----------------
 | Argument | Type | Default | Description |
 | --- | --- | --- | --- |
-| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical. |
+| `K` | `int` | `60` | A constant used in the RRF formula (default is 60). Experiments indicate that k = 60 was near-optimal, but that the choice is not critical |
 | `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score`. If "all", will return all scores from the vector and FTS search along with the relevance score. |
 
 
@@ -49,5 +49,5 @@ You can specify the type of scores you want the reranker to return. The followin
 ### Hybrid Search
 |`return_score`| Status | Description |
 | --- | --- | --- |
-| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column. |
-| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`). |
+| `relevance` | ✅ Supported | Returned rows only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returned rows have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/voyageai.md

@@ -2,7 +2,7 @@
 
 Voyage AI provides cutting-edge embedding and rerankers.
 
-This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker.
+This re-ranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this re-ranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
 
 
 !!! note

docs/src/search.md

@@ -44,16 +44,18 @@ db.create_table("my_vectors", data=data)
 
 === "Python"
 
-    === "Sync API"
+    ```python
+    import lancedb
+    import numpy as np
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search"
-        ```
-    === "Async API"
+    db = lancedb.connect("data/sample-lancedb")
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_async"
-        ```
+    tbl = db.open_table("my_vectors")
+
+    df = tbl.search(np.random.random((1536))) \
+        .limit(10) \
+        .to_list()
+    ```
 
 === "TypeScript"
 
@@ -79,16 +81,12 @@ By default, `l2` will be used as metric type. You can specify the metric type as
 
 === "Python"
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_cosine"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:exhaustive_search_async_cosine"
-        ```
+    ```python
+    df = tbl.search(np.random.random((1536))) \
+        .metric("cosine") \
+        .limit(10) \
+        .to_list()
+    ```
 
 === "TypeScript"
 
@@ -138,98 +136,46 @@ LanceDB supports binary vectors as a data type, and has the ability to search bi
         --8<-- "python/python/tests/docs/test_binary_vector.py:async_binary_vector"
         ```
 
-## Multivector type
-
-LanceDB supports multivector type, this is useful when you have multiple vectors for a single item (e.g. with ColBert and ColPali).
-
-You can index on a column with multivector type and search on it, the query can be single vector or multiple vectors. If the query is multiple vectors `mq`, the similarity (distance) from it to any multivector `mv` in the dataset, is defined as:
-
-![maxsim](assets/maxsim.png)
-
-where `sim` is the similarity function (e.g. cosine).
-
-For now, only `cosine` metric is supported for multivector search.
-The vector value type can be `float16`, `float32` or `float64`.
-
-=== "Python"
-
-    === "sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_multivector.py:imports"
-
-        --8<-- "python/python/tests/docs/test_multivector.py:sync_multivector"
-        ```
-
-    === "async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_multivector.py:imports"
-
-        --8<-- "python/python/tests/docs/test_multivector.py:async_multivector"
-        ```
-
-## Search with distance range
-
-You can also search for vectors within a specific distance range from the query vector. This is useful when you want to find vectors that are not just the nearest neighbors, but also those that are within a certain distance. This can be done by using the `distance_range` method.
-
-=== "Python"
-
-    === "sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_distance_range.py:imports"
-
-        --8<-- "python/python/tests/docs/test_distance_range.py:sync_distance_range"
-        ```
-
-    === "async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_distance_range.py:imports"
-
-        --8<-- "python/python/tests/docs/test_distance_range.py:async_distance_range"
-        ```
-
-=== "TypeScript"
-
-    === "@lancedb/lancedb"
-
-        ```ts
-        --8<-- "nodejs/examples/search.test.ts:import"
-
-        --8<-- "nodejs/examples/search.test.ts:distance_range"
-        ```
-
-
 ## Output search results
 
 LanceDB returns vector search results via different formats commonly used in python.
 Let's create a LanceDB table with a nested schema:
 
 === "Python"
-    === "Sync API"
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_search.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_search.py:import-pydantic-base-model"
-        --8<-- "python/python/tests/docs/test_search.py:class-definition"
-        --8<-- "python/python/tests/docs/test_search.py:create_table_with_nested_schema"
-        ```
-    === "Async API"
+    ```python
 
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_search.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_search.py:import-pydantic-base-model"
-        --8<-- "python/python/tests/docs/test_search.py:class-definition"
-        --8<-- "python/python/tests/docs/test_search.py:create_table_async_with_nested_schema"
-        ```
+    from datetime import datetime
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    import numpy as np
+    from pydantic import BaseModel
+    uri = "data/sample-lancedb-nested"
+
+    class Metadata(BaseModel):
+        source: str
+        timestamp: datetime
+
+    class Document(BaseModel):
+        content: str
+        meta: Metadata
+
+    class LanceSchema(LanceModel):
+        id: str
+        vector: Vector(1536)
+        payload: Document
+
+    # Let's add 100 sample rows to our dataset
+    data = [LanceSchema(
+        id=f"id{i}",
+        vector=np.random.randn(1536),
+        payload=Document(
+            content=f"document{i}", meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now())
+        ),
+    ) for i in range(100)]
+
+    tbl = db.create_table("documents", data=data)
+    ```
 
     ### As a PyArrow table
 
@@ -238,31 +184,17 @@ Let's create a LanceDB table with a nested schema:
     the addition of an `_distance` column for vector search or a `score`
     column for full text search.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pyarrow"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_pyarrow"
-        ```
+    ```python
+    tbl.search(np.random.randn(1536)).to_arrow()
+    ```
 
     ### As a Pandas DataFrame
 
     You can also get the results as a pandas dataframe.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_pandas"
-        ```
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas()
+    ```
 
     While other formats like Arrow/Pydantic/Python dicts have a natural
     way to handle nested schemas, pandas can only store nested data as a
@@ -270,50 +202,33 @@ Let's create a LanceDB table with a nested schema:
     So for convenience, you can also tell LanceDB to flatten a nested schema
     when creating the pandas dataframe.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas_flatten_true"
-        ```
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+    ```
 
     If your table has a deeply nested struct, you can control how many levels
     of nesting to flatten by passing in a positive integer.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pandas_flatten_1"
-        ```
-    !!! note
-        `flatten` is not yet supported with our asynchronous client.
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+    ```
 
     ### As a list of Python dicts
 
     You can of course return results as a list of python dicts.
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_list"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_async_as_list"
-        ```
+    ```python
+    tbl.search(np.random.randn(1536)).to_list()
+    ```
 
     ### As a list of Pydantic models
 
     We can add data using Pydantic models, and we can certainly
     retrieve results as Pydantic models
 
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:search_result_as_pydantic"
-        ```
-    !!! note
-        `to_pydantic()` is not yet supported with our asynchronous client.
+    ```python
+    tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+    ```
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.

docs/src/sql.md

@@ -4,8 +4,8 @@
 
 LanceDB supports filtering of query results based on metadata fields. By default, post-filtering is
 performed on the top-k results returned by the vector search. However, pre-filtering is also an
-option that performs the filter prior to vector search. This can be useful to narrow down
-the search space of a very large dataset to reduce query latency.
+option that performs the filter prior to vector search. This can be useful to narrow down on
+the search space on a very large dataset to reduce query latency.
 
 Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
 
@@ -15,18 +15,13 @@ Similarly, a highly selective post-filter can lead to false positives. Increasin
 ```python
 import lancedb
 import numpy as np
-
 uri = "data/sample-lancedb"
-data = [{"vector": row, "item": f"item {i}", "id": i}
-    for i, row in enumerate(np.random.random((10_000, 2)))]
-
-# Synchronous client
 db = lancedb.connect(uri)
-tbl = db.create_table("my_vectors", data=data)
 
-# Asynchronous client
-async_db = await lancedb.connect_async(uri)
-async_tbl = await async_db.create_table("my_vectors_async", data=data)
+data = [{"vector": row, "item": f"item {i}", "id": i}
+     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+
+tbl = db.create_table("my_vectors", data=data)
 ```
 -->
 <!-- Setup Code
@@ -44,11 +39,13 @@ const tbl = await db.createTable('myVectors', data)
 
 === "Python"
 
-    ```python
-    # Synchronous client
-    result = tbl.search([0.5, 0.2]).where("id = 10", prefilter=True).limit(1).to_arrow()
-    # Asynchronous client
-    result = await async_tbl.query().where("id = 10").nearest_to([0.5, 0.2]).limit(1).to_arrow()
+    ```py
+    result = (
+        tbl.search([0.5, 0.2])
+        .where("id = 10", prefilter=True)
+        .limit(1)
+        .to_arrow()
+    )
     ```
 
 === "TypeScript"
@@ -66,15 +63,15 @@ const tbl = await db.createTable('myVectors', data)
         ```
 !!! note
 
-    Creating a [scalar index](guides/scalar_index.md) accelerates filtering.
+    Creating a [scalar index](guides/scalar_index.md) accelerates filtering
 
 ## SQL filters
 
 Because it's built on top of [DataFusion](https://github.com/apache/arrow-datafusion), LanceDB
 embraces the utilization of standard SQL expressions as predicates for filtering operations.
-SQL can be used during vector search, update, and deletion operations.
+It can be used during vector search, update, and deletion operations.
 
-LanceDB supports a growing list of SQL expressions:
+Currently, Lance supports a growing list of SQL expressions.
 
 - `>`, `>=`, `<`, `<=`, `=`
 - `AND`, `OR`, `NOT`
@@ -91,17 +88,9 @@ For example, the following filter string is acceptable:
 === "Python"
 
     ```python
-    # Synchronous client
-    tbl.search([100, 102]).where(
-        "(item IN ('item 0', 'item 2')) AND (id > 10)"
-    ).to_arrow()
-    # Asynchronous client
-    await (
-        async_tbl.query()
-        .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-        .nearest_to([100, 102])
-        .to_arrow()
-    )
+    tbl.search([100, 102]) \
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
+       .to_arrow()
     ```
 
 === "TypeScript"
@@ -132,7 +121,7 @@ path must be wrapped in backticks.
 !!!warning "Field names containing periods (`.`) are not supported."
 
 Literals for dates, timestamps, and decimals can be written by writing the string
-value after the type name. For example:
+value after the type name. For example
 
 === "SQL"
 
@@ -174,15 +163,12 @@ The mapping from SQL types to Arrow types is:
 
 ## Filtering without Vector Search
 
-You can also filter your data without search:
+You can also filter your data without search.
 
 === "Python"
 
     ```python
-    # Synchronous client
     tbl.search().where("id = 10").limit(10).to_arrow()
-    # Asynchronous client
-    await async_tbl.query().where("id = 10").limit(10).to_arrow()
     ```
 
 === "TypeScript"

docs/test/md_testing.py

@@ -12,8 +12,6 @@ excluded_globs = [
     "../src/integrations/*.md",
     "../src/guides/tables.md",
     "../src/python/duckdb.md",
-    "../src/python/pandas_and_pyarrow.md",
-    "../src/python/polars_arrow.md",
     "../src/embeddings/*.md",
     "../src/concepts/*.md",
     "../src/ann_indexes.md",
@@ -25,10 +23,9 @@ excluded_globs = [
     "../src/embeddings/available_embedding_models/text_embedding_functions/*.md",
     "../src/embeddings/available_embedding_models/multimodal_embedding_functions/*.md",
     "../src/rag/*.md",
-    "../src/rag/advanced_techniques/*.md",
-    "../src/guides/scalar_index.md",
-    "../src/guides/storage.md",
-    "../src/search.md"
+    "../src/rag/advanced_techniques/*.md"
+
+
 ]
 
 python_prefix = "py"

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.15.1-beta.0</version>
+        <version>0.14.1-final.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.15.1-beta.0</version>
+    <version>0.14.1-final.0</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/README.md

@@ -2,9 +2,6 @@
 
 A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
 
-**DEPRECATED: This library is deprecated. Please use the new client,
-[@lancedb/lancedb](https://www.npmjs.com/package/@lancedb/lancedb).**
-
 ## Installation
 
 ```bash

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.15.1-beta.0",
+  "version": "0.14.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.15.1-beta.0",
+      "version": "0.14.1",
       "cpu": [
         "x64",
         "arm64"
@@ -52,14 +52,14 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.15.1-beta.0",
-        "@lancedb/vectordb-darwin-x64": "0.15.1-beta.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.15.1-beta.0",
-        "@lancedb/vectordb-linux-arm64-musl": "0.15.1-beta.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.15.1-beta.0",
-        "@lancedb/vectordb-linux-x64-musl": "0.15.1-beta.0",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.15.1-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.15.1-beta.0"
+        "@lancedb/vectordb-darwin-arm64": "0.14.1",
+        "@lancedb/vectordb-darwin-x64": "0.14.1",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.14.1",
+        "@lancedb/vectordb-linux-arm64-musl": "0.14.1",
+        "@lancedb/vectordb-linux-x64-gnu": "0.14.1",
+        "@lancedb/vectordb-linux-x64-musl": "0.14.1",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.14.1",
+        "@lancedb/vectordb-win32-x64-msvc": "0.14.1"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.15.1-beta.0",
+  "version": "0.14.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "private": false,
   "main": "dist/index.js",
@@ -92,13 +92,13 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.15.1-beta.0",
-    "@lancedb/vectordb-darwin-arm64": "0.15.1-beta.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.15.1-beta.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.15.1-beta.0",
-    "@lancedb/vectordb-linux-x64-musl": "0.15.1-beta.0",
-    "@lancedb/vectordb-linux-arm64-musl": "0.15.1-beta.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.15.1-beta.0",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.15.1-beta.0"
+    "@lancedb/vectordb-darwin-x64": "0.14.1",
+    "@lancedb/vectordb-darwin-arm64": "0.14.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.14.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.14.1",
+    "@lancedb/vectordb-linux-x64-musl": "0.14.1",
+    "@lancedb/vectordb-linux-arm64-musl": "0.14.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.14.1",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.14.1"
   }
 }

nodejs/CONTRIBUTING.md

@@ -1,76 +0,0 @@
-# Contributing to LanceDB Typescript
-
-This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
-
-## Project layout
-
-The Typescript package is a wrapper around the Rust library, `lancedb`. We use
-the [napi-rs](https://napi.rs/) library to create the bindings between Rust and
-Typescript.
-
-* `src/`: Rust bindings source code
-* `lancedb/`: Typescript package source code
-* `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
-
-## Development environment
-
-To set up your development environment, you will need to install the following:
-
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
-
-Initial setup:
-
-```shell
-npm install
-```
-
-### Commit Hooks
-
-It is **highly recommended** to install the [pre-commit](https://pre-commit.com/) hooks to ensure that your
-code is formatted correctly and passes basic checks before committing:
-
-```shell
-pre-commit install
-```
-
-## Development
-
-Most common development commands can be run using the npm scripts.
-
-Build the package
-
-```shell
-npm install
-npm run build
-```
-
-Lint:
-
-```shell
-npm run lint
-```
-
-Format and fix lints:
-
-```shell
-npm run lint-fix
-```
-
-Run tests:
-
-```shell
-npm test
-```
-
-To run a single test:
-
-```shell
-# Single file: table.test.ts
-npm test -- table.test.ts
-# Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
-```

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.15.1-beta.0"
+version = "0.14.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -12,10 +12,7 @@ categories.workspace = true
 crate-type = ["cdylib"]
 
 [dependencies]
-async-trait.workspace = true
 arrow-ipc.workspace = true
-arrow-array.workspace = true
-arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb", features = ["remote"] }

nodejs/README.md

@@ -36,4 +36,37 @@ The [quickstart](../basic.md) contains a more complete example.
 
 ## Development
 
-See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute to LanceDB.
+```sh
+npm run build
+npm run test
+```
+
+### Running lint / format
+
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
+To manually lint your code you can run:
+
+```sh
+npm run lint
+```
+
+to automatically fix all fixable issues:
+
+```sh
+npm run lint-fix
+```
+
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
+
+### Generating docs
+
+```sh
+npm run docs
+
+cd ../docs
+# Asssume the virtual environment was created
+# python3 -m venv venv
+# pip install -r requirements.txt
+. ./venv/bin/activate
+mkdocs build
+```

nodejs/__test__/arrow.test.ts

@@ -20,8 +20,6 @@ import * as arrow18 from "apache-arrow-18";
 
 import {
   convertToTable,
-  fromBufferToRecordBatch,
-  fromRecordBatchToBuffer,
   fromTableToBuffer,
   makeArrowTable,
   makeEmptyTable,
@@ -555,28 +553,5 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         });
       });
     });
-
-    describe("converting record batches to buffers", function () {
-      it("can convert to buffered record batch and back again", async function () {
-        const records = [
-          { text: "dog", vector: [0.1, 0.2] },
-          { text: "cat", vector: [0.3, 0.4] },
-        ];
-        const table = await convertToTable(records);
-        const batch = table.batches[0];
-
-        const buffer = await fromRecordBatchToBuffer(batch);
-        const result = await fromBufferToRecordBatch(buffer);
-
-        expect(JSON.stringify(batch.toArray())).toEqual(
-          JSON.stringify(result?.toArray()),
-        );
-      });
-
-      it("converting from buffer returns null if buffer has no record batches", async function () {
-        const result = await fromBufferToRecordBatch(Buffer.from([0x01, 0x02])); // bad data
-        expect(result).toEqual(null);
-      });
-    });
   },
 );

nodejs/__test__/embedding.test.ts

@@ -83,74 +83,6 @@ describe("embedding functions", () => {
     expect(vector0).toEqual([1, 2, 3]);
   });
 
-  it("should be able to append and upsert using embedding function", async () => {
-    @register()
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      toJSON(): object {
-        return {};
-      }
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): Float {
-        return new Float32();
-      }
-      async computeQueryEmbeddings(_data: string) {
-        return [1, 2, 3];
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return Array.from({ length: data.length }).fill([
-          1, 2, 3,
-        ]) as number[][];
-      }
-    }
-    const func = new MockEmbeddingFunction();
-    const db = await connect(tmpDir.name);
-    const table = await db.createTable(
-      "test",
-      [
-        { id: 1, text: "hello" },
-        { id: 2, text: "world" },
-      ],
-      {
-        embeddingFunction: {
-          function: func,
-          sourceColumn: "text",
-        },
-      },
-    );
-
-    const schema = await table.schema();
-    expect(schema.metadata.get("embedding_functions")).toBeDefined();
-
-    // Append some new data
-    const data1 = [
-      { id: 3, text: "forest" },
-      { id: 4, text: "mountain" },
-    ];
-    await table.add(data1);
-
-    // Upsert some data
-    const data2 = [
-      { id: 5, text: "river" },
-      { id: 2, text: "canyon" },
-    ];
-    await table
-      .mergeInsert("id")
-      .whenMatchedUpdateAll()
-      .whenNotMatchedInsertAll()
-      .execute(data2);
-
-    const rows = await table.query().toArray();
-    rows.sort((a, b) => a.id - b.id);
-    const texts = rows.map((row) => row.text);
-    expect(texts).toEqual(["hello", "canyon", "forest", "mountain", "river"]);
-    const vectorsDefined = rows.map(
-      (row) => row.vector !== undefined && row.vector !== null,
-    );
-    expect(vectorsDefined).toEqual(new Array(5).fill(true));
-  });
-
   it("should be able to create an empty table with an embedding function", async () => {
     @register()
     class MockEmbeddingFunction extends EmbeddingFunction<string> {

nodejs/__test__/rerankers.test.ts

@@ -1,79 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import { RecordBatch } from "apache-arrow";
-import * as tmp from "tmp";
-import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
-import { RRFReranker } from "../lancedb/rerankers";
-
-describe("rerankers", function () {
-  let tmpDir: tmp.DirResult;
-  let conn: Connection;
-  let table: Table;
-
-  beforeEach(async () => {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    conn = await connect(tmpDir.name);
-    table = await conn.createTable("mytable", [
-      { vector: [0.1, 0.1], text: "dog" },
-      { vector: [0.2, 0.2], text: "cat" },
-    ]);
-    await table.createIndex("text", {
-      config: Index.fts(),
-      replace: true,
-    });
-  });
-
-  it("will query with the custom reranker", async function () {
-    const expectedResult = [
-      {
-        text: "albert",
-        // biome-ignore lint/style/useNamingConvention: this is the lance field name
-        _relevance_score: 0.99,
-      },
-    ];
-    class MyCustomReranker {
-      async rerankHybrid(
-        _query: string,
-        _vecResults: RecordBatch,
-        _ftsResults: RecordBatch,
-      ): Promise<RecordBatch> {
-        // no reranker logic, just return some static data
-        const table = makeArrowTable(expectedResult);
-        return table.batches[0];
-      }
-    }
-
-    let result = await table
-      .query()
-      .nearestTo([0.1, 0.1])
-      .fullTextSearch("dog")
-      .rerank(new MyCustomReranker())
-      .select(["text"])
-      .limit(5)
-      .toArray();
-
-    result = JSON.parse(JSON.stringify(result)); // convert StructRow to Object
-    expect(result).toEqual([
-      {
-        text: "albert",
-        // biome-ignore lint/style/useNamingConvention: this is the lance field name
-        _relevance_score: 0.99,
-      },
-    ]);
-  });
-
-  it("will query with RRFReranker", async function () {
-    // smoke test to see if the Rust wrapping Typescript is wired up correctly
-    const result = await table
-      .query()
-      .nearestTo([0.1, 0.1])
-      .fullTextSearch("dog")
-      .rerank(await RRFReranker.create())
-      .select(["text"])
-      .limit(5)
-      .toArray();
-
-    expect(result).toHaveLength(2);
-  });
-});

nodejs/__test__/table.test.ts

@@ -473,66 +473,6 @@ describe("When creating an index", () => {
     // test offset
     rst = await tbl.query().limit(2).offset(1).nearestTo(queryVec).toArrow();
     expect(rst.numRows).toBe(1);
-
-    await tbl.dropIndex("vec_idx");
-    const indices2 = await tbl.listIndices();
-    expect(indices2.length).toBe(0);
-  });
-
-  it("should search with distance range", async () => {
-    await tbl.createIndex("vec");
-
-    const rst = await tbl.query().limit(10).nearestTo(queryVec).toArrow();
-    const distanceColumn = rst.getChild("_distance");
-    let minDist = undefined;
-    let maxDist = undefined;
-    if (distanceColumn) {
-      minDist = distanceColumn.get(0);
-      maxDist = distanceColumn.get(9);
-    }
-
-    const rst2 = await tbl
-      .query()
-      .limit(10)
-      .nearestTo(queryVec)
-      .distanceRange(minDist, maxDist)
-      .toArrow();
-    const distanceColumn2 = rst2.getChild("_distance");
-    expect(distanceColumn2).toBeDefined();
-    if (distanceColumn2) {
-      for await (const d of distanceColumn2) {
-        expect(d).toBeGreaterThanOrEqual(minDist);
-        expect(d).toBeLessThan(maxDist);
-      }
-    }
-
-    const rst3 = await tbl
-      .query()
-      .limit(10)
-      .nearestTo(queryVec)
-      .distanceRange(maxDist, undefined)
-      .toArrow();
-    const distanceColumn3 = rst3.getChild("_distance");
-    expect(distanceColumn3).toBeDefined();
-    if (distanceColumn3) {
-      for await (const d of distanceColumn3) {
-        expect(d).toBeGreaterThanOrEqual(maxDist);
-      }
-    }
-
-    const rst4 = await tbl
-      .query()
-      .limit(10)
-      .nearestTo(queryVec)
-      .distanceRange(undefined, minDist)
-      .toArrow();
-    const distanceColumn4 = rst4.getChild("_distance");
-    expect(distanceColumn4).toBeDefined();
-    if (distanceColumn4) {
-      for await (const d of distanceColumn4) {
-        expect(d).toBeLessThan(minDist);
-      }
-    }
   });
 
   it("should create and search IVF_HNSW indices", async () => {
@@ -771,9 +711,7 @@ describe("When creating an index", () => {
         )
         .column("vec")
         .toArrow(),
-    ).rejects.toThrow(
-      /.* query dim\(64\) doesn't match the column vec vector dim\(32\).*/,
-    );
+    ).rejects.toThrow(/.* query dim=64, expected vector dim=32.*/);
 
     const query64 = Array(64)
       .fill(1)

nodejs/examples/search.test.ts

@@ -38,19 +38,5 @@ test("full text search", async () => {
       .toArray();
     // --8<-- [end:search2]
     expect(results2.length).toBe(10);
-
-    // --8<-- [start:distance_range]
-    const results3 = await (
-      tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
-    )
-      .distanceType("cosine")
-      .distanceRange(0.1, 0.2)
-      .limit(10)
-      .toArray();
-    // --8<-- [end:distance_range]
-    for (const r of results3) {
-      expect(r.distance).toBeGreaterThanOrEqual(0.1);
-      expect(r.distance).toBeLessThan(0.2);
-    }
   });
 });

nodejs/lancedb/arrow.ts

@@ -27,9 +27,7 @@ import {
   List,
   Null,
   RecordBatch,
-  RecordBatchFileReader,
   RecordBatchFileWriter,
-  RecordBatchReader,
   RecordBatchStreamWriter,
   Schema,
   Struct,
@@ -609,14 +607,6 @@ async function applyEmbeddings<T>(
     return table;
   }
 
-  let schemaMetadata = schema?.metadata || new Map<string, string>();
-
-  if (!(embeddings == null || embeddings === undefined)) {
-    const registry = getRegistry();
-    const embeddingMetadata = registry.getTableMetadata([embeddings]);
-    schemaMetadata = new Map([...schemaMetadata, ...embeddingMetadata]);
-  }
-
   // Convert from ArrowTable to Record<String, Vector>
   const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
     const name = table.schema.fields[idx].name;
@@ -685,21 +675,15 @@ async function applyEmbeddings<T>(
     newColumns[destColumn] = makeVector(vectors, destType);
   }
 
-  let newTable = new ArrowTable(newColumns);
+  const newTable = new ArrowTable(newColumns);
   if (schema != null) {
     if (schema.fields.find((f) => f.name === destColumn) === undefined) {
       throw new Error(
         `When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`,
       );
     }
-    newTable = alignTable(newTable, schema as Schema);
+    return alignTable(newTable, schema as Schema);
   }
-
-  newTable = new ArrowTable(
-    new Schema(newTable.schema.fields, schemaMetadata),
-    newTable.batches,
-  );
-
   return newTable;
 }
 
@@ -826,30 +810,6 @@ export async function fromDataToBuffer(
   }
 }
 
-/**
- * Read a single record batch from a buffer.
- *
- * Returns null if the buffer does not contain a record batch
- */
-export async function fromBufferToRecordBatch(
-  data: Buffer,
-): Promise<RecordBatch | null> {
-  const iter = await RecordBatchFileReader.readAll(Buffer.from(data)).next()
-    .value;
-  const recordBatch = iter?.next().value;
-  return recordBatch || null;
-}
-
-/**
- * Create a buffer containing a single record batch
- */
-export async function fromRecordBatchToBuffer(
-  batch: RecordBatch,
-): Promise<Buffer> {
-  const writer = new RecordBatchFileWriter().writeAll([batch]);
-  return Buffer.from(await writer.toUint8Array());
-}
-
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

nodejs/lancedb/index.ts

@@ -62,7 +62,6 @@ export { Index, IndexOptions, IvfPqOptions } from "./indices";
 export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 
 export * as embedding from "./embedding";
-export * as rerankers from "./rerankers";
 
 /**
  * Connect to a LanceDB instance at the given URI.

nodejs/lancedb/merge.ts

@@ -1,20 +1,13 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-import { Data, Schema, fromDataToBuffer } from "./arrow";
+import { Data, fromDataToBuffer } from "./arrow";
 import { NativeMergeInsertBuilder } from "./native";
 
 /** A builder used to create and run a merge insert operation */
 export class MergeInsertBuilder {
   #native: NativeMergeInsertBuilder;
-  #schema: Schema | Promise<Schema>;
 
   /** Construct a MergeInsertBuilder. __Internal use only.__ */
-  constructor(
-    native: NativeMergeInsertBuilder,
-    schema: Schema | Promise<Schema>,
-  ) {
+  constructor(native: NativeMergeInsertBuilder) {
     this.#native = native;
-    this.#schema = schema;
   }
 
   /**
@@ -42,7 +35,6 @@ export class MergeInsertBuilder {
   whenMatchedUpdateAll(options?: { where: string }): MergeInsertBuilder {
     return new MergeInsertBuilder(
       this.#native.whenMatchedUpdateAll(options?.where),
-      this.#schema,
     );
   }
   /**
@@ -50,10 +42,7 @@ export class MergeInsertBuilder {
    * be inserted into the target table.
    */
   whenNotMatchedInsertAll(): MergeInsertBuilder {
-    return new MergeInsertBuilder(
-      this.#native.whenNotMatchedInsertAll(),
-      this.#schema,
-    );
+    return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
   }
   /**
    * Rows that exist only in the target table (old data) will be
@@ -67,7 +56,6 @@ export class MergeInsertBuilder {
   }): MergeInsertBuilder {
     return new MergeInsertBuilder(
       this.#native.whenNotMatchedBySourceDelete(options?.where),
-      this.#schema,
     );
   }
   /**
@@ -76,14 +64,7 @@ export class MergeInsertBuilder {
    * Nothing is returned but the `Table` is updated
    */
   async execute(data: Data): Promise<void> {
-    let schema: Schema;
-    if (this.#schema instanceof Promise) {
-      schema = await this.#schema;
-      this.#schema = schema; // In case of future calls
-    } else {
-      schema = this.#schema;
-    }
-    const buffer = await fromDataToBuffer(data, undefined, schema);
+    const buffer = await fromDataToBuffer(data);
     await this.#native.execute(buffer);
   }
 }

nodejs/lancedb/query.ts

@@ -16,8 +16,6 @@ import {
   Table as ArrowTable,
   type IntoVector,
   RecordBatch,
-  fromBufferToRecordBatch,
-  fromRecordBatchToBuffer,
   tableFromIPC,
 } from "./arrow";
 import { type IvfPqOptions } from "./indices";
@@ -27,7 +25,6 @@ import {
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
-import { Reranker } from "./rerankers";
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -388,19 +385,6 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
     return this;
   }
 
-  /*
-   * Set the distance range to use
-   *
-   * Only rows with distances within range [lower_bound, upper_bound)
-   * will be returned.
-   *
-   * `undefined` means no lower or upper bound.
-   */
-  distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
-    super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
-    return this;
-  }
-
   /**
    * Set the number of candidates to consider during the search
    *
@@ -558,27 +542,6 @@ export class VectorQuery extends QueryBase<NativeVectorQuery> {
       return this;
     }
   }
-
-  rerank(reranker: Reranker): VectorQuery {
-    super.doCall((inner) =>
-      inner.rerank({
-        rerankHybrid: async (_, args) => {
-          const vecResults = await fromBufferToRecordBatch(args.vecResults);
-          const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
-          const result = await reranker.rerankHybrid(
-            args.query,
-            vecResults as RecordBatch,
-            ftsResults as RecordBatch,
-          );
-
-          const buffer = fromRecordBatchToBuffer(result);
-          return buffer;
-        },
-      }),
-    );
-
-    return this;
-  }
 }
 
 /** A builder for LanceDB queries. */

nodejs/lancedb/rerankers/index.ts

@@ -1,17 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import { RecordBatch } from "apache-arrow";
-
-export * from "./rrf";
-
-// Interface for a reranker. A reranker is used to rerank the results from a
-// vector and FTS search. This is useful for combining the results from both
-// search methods.
-export interface Reranker {
-  rerankHybrid(
-    query: string,
-    vecResults: RecordBatch,
-    ftsResults: RecordBatch,
-  ): Promise<RecordBatch>;
-}

nodejs/lancedb/rerankers/rrf.ts

@@ -1,40 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import { RecordBatch } from "apache-arrow";
-import { fromBufferToRecordBatch, fromRecordBatchToBuffer } from "../arrow";
-import { RrfReranker as NativeRRFReranker } from "../native";
-
-/**
- * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
- *
- * Internally this uses the Rust implementation
- */
-export class RRFReranker {
-  private inner: NativeRRFReranker;
-
-  constructor(inner: NativeRRFReranker) {
-    this.inner = inner;
-  }
-
-  public static async create(k: number = 60) {
-    return new RRFReranker(
-      await NativeRRFReranker.tryNew(new Float32Array([k])),
-    );
-  }
-
-  async rerankHybrid(
-    query: string,
-    vecResults: RecordBatch,
-    ftsResults: RecordBatch,
-  ): Promise<RecordBatch> {
-    const buffer = await this.inner.rerankHybrid(
-      query,
-      await fromRecordBatchToBuffer(vecResults),
-      await fromRecordBatchToBuffer(ftsResults),
-    );
-    const recordBatch = await fromBufferToRecordBatch(buffer);
-
-    return recordBatch as RecordBatch;
-  }
-}

nodejs/lancedb/table.ts

@@ -226,19 +226,6 @@ export abstract class Table {
     column: string,
     options?: Partial<IndexOptions>,
   ): Promise<void>;
-
-  /**
-   * Drop an index from the table.
-   *
-   * @param name The name of the index.
-   *
-   * @note This does not delete the index from disk, it just removes it from the table.
-   * To delete the index, run {@link Table#optimize} after dropping the index.
-   *
-   * Use {@link Table.listIndices} to find the names of the indices.
-   */
-  abstract dropIndex(name: string): Promise<void>;
-
   /**
    * Create a {@link Query} Builder.
    *
@@ -439,8 +426,6 @@ export abstract class Table {
    *
    * @param {string} name The name of the index.
    * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
-   *
-   * Use {@link Table.listIndices} to find the names of the indices.
    */
   abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
 
@@ -520,8 +505,14 @@ export class LocalTable extends Table {
   async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
     const mode = options?.mode ?? "append";
     const schema = await this.schema();
+    const registry = getRegistry();
+    const functions = await registry.parseFunctions(schema.metadata);
 
-    const buffer = await fromDataToBuffer(data, undefined, schema);
+    const buffer = await fromDataToBuffer(
+      data,
+      functions.values().next().value,
+      schema,
+    );
     await this.inner.add(buffer, mode);
   }
 
@@ -600,10 +591,6 @@ export class LocalTable extends Table {
     await this.inner.createIndex(nativeIndex, column, options?.replace);
   }
 
-  async dropIndex(name: string): Promise<void> {
-    await this.inner.dropIndex(name);
-  }
-
   query(): Query {
     return new Query(this.inner);
   }
@@ -727,7 +714,7 @@ export class LocalTable extends Table {
   }
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
-    return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
+    return new MergeInsertBuilder(this.inner.mergeInsert(on));
   }
 
   /**

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.15.1-beta.0",
+  "version": "0.14.1",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.15.1-beta.0",
+	"version": "0.14.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.15.1-beta.0",
+  "version": "0.14.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.15.1-beta.0",
+      "version": "0.14.1",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.15.1-beta.0",
+  "version": "0.14.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/lib.rs

@@ -24,7 +24,6 @@ mod iterator;
 pub mod merge;
 mod query;
 pub mod remote;
-mod rerankers;
 mod table;
 mod util;