docs: Add async api index types

2025-12-23 05:19:58 +00:00 · 2024-11-21 16:26:32 -08:00
400 changed files with 8262 additions and 31579 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.0"
+current_version = "0.13.1-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/.github/workflows/build_linux_wheel/action.yml
+++ b/.github/workflows/build_linux_wheel/action.yml
@@ -52,7 +52,12 @@ runs:
        args: ${{ inputs.args }}
        before-script-linux: |
          set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
            && unzip /tmp/protoc.zip -d /usr/local \
            && rm /tmp/protoc.zip
--- a/.github/workflows/build_mac_wheel/action.yml
+++ b/.github/workflows/build_mac_wheel/action.yml
@@ -20,7 +20,7 @@ runs:
      uses: PyO3/maturin-action@v1
      with:
        command: build
-        # TODO: pass through interpreter
        args: ${{ inputs.args }}
        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
        working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}
--- a/.github/workflows/build_windows_wheel/action.yml
+++ b/.github/workflows/build_windows_wheel/action.yml
@@ -28,7 +28,7 @@ runs:
        args: ${{ inputs.args }}
        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
        working-directory: python
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v3
      with:
        name: windows-wheels
        path: python\target\wheels
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -72,9 +72,9 @@ jobs:
      - name: Setup Pages
        uses: actions/configure-pages@v2
      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v1
        with:
          path: "docs/site"
      - name: Deploy to GitHub Pages
        id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v1
--- a/.github/workflows/license-header-check.yml
+++ b/.github/workflows/license-header-check.yml
@@ -1,31 +0,0 @@
-name: Check license headers
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - rust/**
-      - python/**
-      - nodejs/**
-      - java/**
-      - .github/workflows/license-header-check.yml
-jobs:
-  check-licenses:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-      - name: Install license-header-checker
-        working-directory: /tmp
-        run: |
-            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
-            mv /tmp/bin/license-header-checker /usr/local/bin/
-      - name: Check license headers (rust)
-        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
-      - name: Check license headers (python)
-        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
-      - name: Check license headers (typescript)
-        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
-      - name: Check license headers (java)
-        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]
--- a/.github/workflows/make-release-commit.yml
+++ b/.github/workflows/make-release-commit.yml
@@ -43,7 +43,7 @@ on:
 jobs:
  make-release:
    # Creates tag and GH release. The GH release will trigger the build and release jobs.
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
@@ -57,14 +57,15 @@ jobs:
          # trigger any workflows watching for new tags. See:
          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-      - name: Validate Lance dependency is at stable version
-        if: ${{ inputs.type == 'stable' }}
-        run: python ci/validate_stable_lance.py
      - name: Set git configs for bumpversion
        shell: bash
        run: |
          git config user.name 'Lance Release'
          git config user.email 'lance-dev@lancedb.com'
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
      - name: Bump Python version
        if: ${{ inputs.python }}
        working-directory: python
@@ -96,7 +97,3 @@ jobs:
        if: ${{ !inputs.dry_run && inputs.other }}
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        if: ${{ !inputs.dry_run && inputs.other }}
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
--- a/.github/workflows/nodejs.yml
+++ b/.github/workflows/nodejs.yml
@@ -106,18 +106,6 @@ jobs:
        python ci/mock_openai.py &
        cd nodejs/examples
        npm test
-    - name: Check docs
-      run: |
-        # We run this as part of the job because the binary needs to be built
-        # first to export the types of the native code.
-        set -e
-        npm ci
-        npm run docs
-        if ! git diff --exit-code; then
-          echo "Docs need to be updated"
-          echo "Run 'npm run docs', fix any warnings, and commit the changes."
-          exit 1
-        fi
  macos:
    timeout-minutes: 30
    runs-on: "macos-14"
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -133,7 +133,7 @@ jobs:
          free -h
      - name: Build Linux Artifacts
        run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
@@ -159,7 +159,7 @@ jobs:
      - name: Install common dependencies
        run: |
          apk add protobuf-dev curl clang mold grep npm bash
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
          echo "source $HOME/.cargo/env" >> saved_env
          echo "export CC=clang" >> saved_env
          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
@@ -167,7 +167,7 @@ jobs:
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
          source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl
+          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
@@ -185,7 +185,7 @@ jobs:
      - name: Build Linux Artifacts
        run: |
          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-musl
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
@@ -262,7 +262,7 @@ jobs:
      - name: Install common dependencies
        run: |
          apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0
          echo "source $HOME/.cargo/env" >> saved_env
          echo "export CC=clang" >> saved_env
          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env
@@ -272,7 +272,7 @@ jobs:
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
          source "$HOME/.cargo/env"
-          rustup target add aarch64-unknown-linux-musl
+          rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0
          crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained)
          sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib
          apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/
@@ -334,50 +334,109 @@ jobs:
          path: |
            node/dist/lancedb-vectordb-win32*.tgz

-  node-windows-arm64:
-    name: vectordb ${{ matrix.config.arch }}-pc-windows-msvc
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-pc-windows-msvc
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: node-native-windows-${{ matrix.config.arch }}
-          path: |
-            node/dist/lancedb-vectordb-win32*.tgz
+  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
+  # node-windows-arm64:
+  #   name: vectordb win32-arm64-msvc
+  #   runs-on: windows-4x-arm
+  #   if: startsWith(github.ref, 'refs/tags/v')
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Install Git
+  #       run: |
+  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+  #       shell: powershell
+  #     - name: Add Git to PATH
+  #       run: |
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+  #       shell: powershell
+  #     - name: Configure Git symlinks
+  #       run: git config --global core.symlinks true
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: "3.13"
+  #     - name: Install Visual Studio Build Tools
+  #       run: |
+  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+  #           "--installPath", "C:\BuildTools", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+  #       shell: powershell
+  #     - name: Add Visual Studio Build Tools to PATH
+  #       run: |
+  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+
+  #         # Add MSVC runtime libraries to LIB
+  #         $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
+  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
+  #                    "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+  #         Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
+
+  #         # Add INCLUDE paths
+  #         $env:INCLUDE = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\include;" +
+  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\ucrt;" +
+  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\um;" +
+  #                       "C:\Program Files (x86)\Windows Kits\10\Include\10.0.22621.0\shared"
+  #         Add-Content $env:GITHUB_ENV "INCLUDE=$env:INCLUDE"
+  #       shell: powershell
+  #     - name: Install Rust
+  #       run: |
+  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+  #       shell: powershell
+  #     - name: Add Rust to PATH
+  #       run: |
+  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+  #       shell: powershell
+
+  #     - uses: Swatinem/rust-cache@v2
+  #       with:
+  #         workspaces: rust
+  #     - name: Install 7-Zip ARM
+  #       run: |
+  #         New-Item -Path 'C:\7zip' -ItemType Directory
+  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+  #       shell: powershell
+  #     - name: Add 7-Zip to PATH
+  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+  #       shell: powershell
+  #     - name: Install Protoc v21.12
+  #       working-directory: C:\
+  #       run: |
+  #         if (Test-Path 'C:\protoc') {
+  #             Write-Host "Protoc directory exists, skipping installation"
+  #             return
+  #         }
+  #         New-Item -Path 'C:\protoc' -ItemType Directory
+  #         Set-Location C:\protoc
+  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+  #       shell: powershell
+  #     - name: Add Protoc to PATH
+  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+  #       shell: powershell
+  #     - name: Build Windows native node modules
+  #       run: .\ci\build_windows_artifacts.ps1 aarch64-pc-windows-msvc
+  #     - name: Upload Windows ARM64 Artifacts
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: node-native-windows-arm64
+  #         path: |
+  #           node/dist/*.node

  nodejs-windows:
    name: lancedb ${{ matrix.target }}
@@ -413,57 +472,103 @@ jobs:
          path: |
            nodejs/dist/*.node

-  nodejs-windows-arm64:
-    name: lancedb ${{ matrix.config.arch }}-pc-windows-msvc
-    # Only runs on tags that matches the make-release action
-    # if: startsWith(github.ref, 'refs/tags/v')
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          # - arch: x86_64
-          - arch: aarch64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies
-        run: |
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
-          echo "source $HOME/.cargo/env" >> saved_env
-          echo "export CC=clang" >> saved_env
-          echo "export AR=llvm-ar" >> saved_env
-          source "$HOME/.cargo/env"
-          rustup target add ${{ matrix.config.arch }}-pc-windows-msvc
-          (mkdir -p sysroot && cd sysroot && sh ../ci/sysroot-${{ matrix.config.arch }}-pc-windows-msvc.sh)
-          echo "export C_INCLUDE_PATH=/usr/${{ matrix.config.arch }}-pc-windows-msvc/usr/include" >> saved_env
-          echo "export CARGO_BUILD_TARGET=${{ matrix.config.arch }}-pc-windows-msvc" >> saved_env
-          printf '#!/bin/sh\ncargo "$@"' > $HOME/.cargo/bin/cargo-xwin
-          chmod u+x $HOME/.cargo/bin/cargo-xwin
-      - name: Configure x86_64 build
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=+crt-static,+avx2,+fma,+f16c -Clinker=lld -Clink-arg=/LIBPATH:/usr/x86_64-pc-windows-msvc/usr/lib'" >> saved_env
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          echo "export RUSTFLAGS='-Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib'" >> saved_env
-      - name: Build Windows Artifacts
-        run: |
-          source ./saved_env
-          bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
+  # TODO: re-enable once working https://github.com/lancedb/lancedb/pull/1831
+  # nodejs-windows-arm64:
+  #   name: lancedb win32-arm64-msvc
+  #   runs-on: windows-4x-arm
+  #   if: startsWith(github.ref, 'refs/tags/v')
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Install Git
+  #       run: |
+  #         Invoke-WebRequest -Uri "https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe" -OutFile "git-installer.exe"
+  #         Start-Process -FilePath "git-installer.exe" -ArgumentList "/VERYSILENT", "/NORESTART" -Wait
+  #       shell: powershell
+  #     - name: Add Git to PATH
+  #       run: |
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files\Git\bin"
+  #         $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")
+  #       shell: powershell
+  #     - name: Configure Git symlinks
+  #       run: git config --global core.symlinks true
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: "3.13"
+  #     - name: Install Visual Studio Build Tools
+  #       run: |
+  #         Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "vs_buildtools.exe"
+  #         Start-Process -FilePath "vs_buildtools.exe" -ArgumentList "--quiet", "--wait", "--norestart", "--nocache", `
+  #           "--installPath", "C:\BuildTools", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.ARM64", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", `
+  #           "--add", "Microsoft.VisualStudio.Component.Windows11SDK.22621", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.ATL", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.ATLMFC", `
+  #           "--add", "Microsoft.VisualStudio.Component.VC.Llvm.Clang" -Wait
+  #       shell: powershell
+  #     - name: Add Visual Studio Build Tools to PATH
+  #       run: |
+  #         $vsPath = "C:\BuildTools\VC\Tools\MSVC"
+  #         $latestVersion = (Get-ChildItem $vsPath | Sort-Object {[version]$_.Name} -Descending)[0].Name
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\arm64"
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\MSVC\$latestVersion\bin\Hostx64\x64"
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\arm64"
+  #         Add-Content $env:GITHUB_PATH "C:\Program Files (x86)\Windows Kits\10\bin\10.0.22621.0\x64"
+  #         Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"
+
+  #         $env:LIB = ""
+  #         Add-Content $env:GITHUB_ENV "LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
+  #       shell: powershell
+  #     - name: Install Rust
+  #       run: |
+  #         Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
+  #         .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
+  #       shell: powershell
+  #     - name: Add Rust to PATH
+  #       run: |
+  #         Add-Content $env:GITHUB_PATH "$env:USERPROFILE\.cargo\bin"
+  #       shell: powershell
+
+  #     - uses: Swatinem/rust-cache@v2
+  #       with:
+  #         workspaces: rust
+  #     - name: Install 7-Zip ARM
+  #       run: |
+  #         New-Item -Path 'C:\7zip' -ItemType Directory
+  #         Invoke-WebRequest https://7-zip.org/a/7z2408-arm64.exe -OutFile C:\7zip\7z-installer.exe
+  #         Start-Process -FilePath C:\7zip\7z-installer.exe -ArgumentList '/S' -Wait
+  #       shell: powershell
+  #     - name: Add 7-Zip to PATH
+  #       run: Add-Content $env:GITHUB_PATH "C:\Program Files\7-Zip"
+  #       shell: powershell
+  #     - name: Install Protoc v21.12
+  #       working-directory: C:\
+  #       run: |
+  #         if (Test-Path 'C:\protoc') {
+  #             Write-Host "Protoc directory exists, skipping installation"
+  #             return
+  #         }
+  #         New-Item -Path 'C:\protoc' -ItemType Directory
+  #         Set-Location C:\protoc
+  #         Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+  #         & 'C:\Program Files\7-Zip\7z.exe' x protoc.zip
+  #       shell: powershell
+  #     - name: Add Protoc to PATH
+  #       run: Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+  #       shell: powershell
+  #     - name: Build Windows native node modules
+  #       run: .\ci\build_windows_artifacts_nodejs.ps1 aarch64-pc-windows-msvc
+  #     - name: Upload Windows ARM64 Artifacts
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: nodejs-native-windows-arm64
+  #         path: |
+  #           nodejs/dist/*.node

  release:
    name: vectordb NPM Publish
-    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows, node-windows-arm64]
+    needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
@@ -503,7 +608,7 @@ jobs:

  release-nodejs:
    name: lancedb NPM Publish
-    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows, nodejs-windows-arm64]
+    needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
@@ -561,7 +666,6 @@ jobs:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  update-package-lock:
-    if: startsWith(github.ref, 'refs/tags/v')
    needs: [release]
    runs-on: ubuntu-latest
    permissions:
@@ -571,7 +675,7 @@ jobs:
        uses: actions/checkout@v4
        with:
          ref: main
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock
@@ -579,7 +683,6 @@ jobs:
          github_token: ${{ secrets.GITHUB_TOKEN }}

  update-package-lock-nodejs:
-    if: startsWith(github.ref, 'refs/tags/v')
    needs: [release-nodejs]
    runs-on: ubuntu-latest
    permissions:
@@ -589,7 +692,7 @@ jobs:
        uses: actions/checkout@v4
        with:
          ref: main
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock_nodejs
@@ -597,7 +700,6 @@ jobs:
          github_token: ${{ secrets.GITHUB_TOKEN }}

  gh-release:
-    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
--- a/.github/workflows/pypi-publish.yml
+++ b/.github/workflows/pypi-publish.yml
@@ -15,21 +15,15 @@ jobs:
          - platform: x86_64
            manylinux: "2_17"
            extra_args: ""
-            runner: ubuntu-22.04
          - platform: x86_64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
-            runner: ubuntu-22.04
          - platform: aarch64
-            manylinux: "2_17"
+            manylinux: "2_24"
            extra_args: ""
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: ubuntu-2404-8x-arm64
-          - platform: aarch64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-            runner: ubuntu-2404-8x-arm64
-    runs-on: ${{ matrix.config.runner }}
+          # We don't build fp16 kernels for aarch64, because it uses 
+          # cross compilation image, which doesn't have a new enough compiler.
+    runs-on: "ubuntu-22.04"
    steps:
      - uses: actions/checkout@v4
        with:
@@ -89,7 +83,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
-          python-version: 3.12
+          python-version: 3.8
      - uses: ./.github/workflows/build_windows_wheel
        with:
          python-minor-version: 8
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -30,17 +30,16 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.12"
+          python-version: "3.11"
      - name: Install ruff
        run: |
-          pip install ruff==0.9.9
+          pip install ruff==0.5.4
      - name: Format check
        run: ruff format --check .
      - name: Lint
        run: ruff check .
-
-  type-check:
-    name: "Type Check"
+  doctest:
+    name: "Doctest"
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
@@ -55,36 +54,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.12"
-      - name: Install protobuf compiler
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-          pip install toml
-      - name: Install dependencies
-        run: |
-          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
-          pip install -r requirements.txt
-      - name: Run pyright
-        run: pyright
-
-  doctest:
-    name: "Doctest"
-    timeout-minutes: 30
-    runs-on: "ubuntu-24.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
+          python-version: "3.11"
          cache: "pip"
      - name: Install protobuf
        run: |
@@ -105,8 +75,8 @@ jobs:
    timeout-minutes: 30
    strategy:
      matrix:
-        python-minor-version: ["9", "12"]
-    runs-on: "ubuntu-24.04"
+        python-minor-version: ["9", "11"]
+    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
@@ -157,7 +127,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.12"
+          python-version: "3.11"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
@@ -187,7 +157,7 @@ jobs:
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: "3.12"
+          python-version: "3.11"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
@@ -198,7 +168,7 @@ jobs:
        run: rm -rf target/wheels
  pydantic1x:
    timeout-minutes: 30
-    runs-on: "ubuntu-24.04"
+    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -22,7 +22,6 @@ env:
  # "1" means line tables only, which is useful for panic tracebacks.
  RUSTFLAGS: "-C debuginfo=1"
  RUST_BACKTRACE: "1"
-  CARGO_INCREMENTAL: 0

 jobs:
  lint:
@@ -52,33 +51,6 @@ jobs:
      - name: Run clippy
        run: cargo clippy --workspace --tests --all-features -- -D warnings

-  build-no-lock:
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang
-      CXX: clang++
-    steps:
-      - uses: actions/checkout@v4
-      # Building without a lock file often requires the latest Rust version since downstream
-      # dependencies may have updated their minimum Rust version.
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          toolchain: "stable"
-      # Remove cargo.lock to force a fresh build
-      - name: Remove Cargo.lock
-        run: rm -f Cargo.lock
-      - uses: rui314/setup-mold@v1
-      - uses: Swatinem/rust-cache@v2
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build all
-        run: |
-          cargo build --benches --all-features --tests
-
  linux:
    timeout-minutes: 30
    # To build all features, we need more disk space than is available
@@ -103,11 +75,8 @@ jobs:
          workspaces: rust
      - name: Install dependencies
        run: |
-          # This shaves 2 minutes off this step in CI. This doesn't seem to be
-          # necessary in standard runners, but it is in the 4x runners.
-          sudo rm /var/lib/man-db/auto-update
+          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
-      - uses: rui314/setup-mold@v1
      - name: Make Swap
        run: |
          sudo fallocate -l 16G /swapfile
@@ -118,11 +87,11 @@ jobs:
        working-directory: .
        run: docker compose up --detach --wait
      - name: Build
-        run: cargo build --all-features --tests --locked --examples
+        run: cargo build --all-features
      - name: Run tests
-        run: cargo test --all-features --locked
+        run: cargo test --all-features
      - name: Run examples
-        run: cargo run --example simple --locked
+        run: cargo run --example simple

  macos:
    timeout-minutes: 30
@@ -146,14 +115,11 @@ jobs:
          workspaces: rust
      - name: Install dependencies
        run: brew install protobuf
+      - name: Build
+        run: cargo build --all-features
      - name: Run tests
-        run: |
-          # Don't run the s3 integration tests since docker isn't available
-          # on this image.
-          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
-            | jq -r '.packages[] | .features | keys | .[]' \
-            | grep -v s3-test | sort | uniq | paste -s -d "," -`
-          cargo test --features $ALL_FEATURES --locked
+        # Run with everything except the integration tests.
+        run: cargo test --features remote,fp16kernels

  windows:
    runs-on: windows-2022
@@ -174,40 +140,8 @@ jobs:
      - name: Run tests
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --features remote --locked
-
-  windows-arm64-cross:
-    # We cross compile in Node releases, so we want to make sure
-    # this can run successfully.
-    runs-on: ubuntu-latest
-    container: alpine:edge
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install dependencies (part 1)
-        run: |
-          set -e
-          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
-      - name: Install rust
-        uses: actions-rust-lang/setup-rust-toolchain@v1
-        with:
-          target: aarch64-pc-windows-msvc
-      - name: Install dependencies (part 2)
-        run: |
-          set -e
-          mkdir -p sysroot
-          cd sysroot
-          sh ../ci/sysroot-aarch64-pc-windows-msvc.sh
-      - name: Check
-        env:
-          CC: clang
-          AR: llvm-ar
-          C_INCLUDE_PATH: /usr/aarch64-pc-windows-msvc/usr/include
-          CARGO_BUILD_TARGET: aarch64-pc-windows-msvc
-          RUSTFLAGS: -Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib
-        run: |
-          source $HOME/.cargo/env
-          cargo check --features remote --locked
+          cargo build
+          cargo test

  windows-arm64:
    runs-on: windows-4x-arm
@@ -251,7 +185,7 @@ jobs:
          Add-Content $env:GITHUB_PATH "C:\BuildTools\VC\Tools\Llvm\x64\bin"

          # Add MSVC runtime libraries to LIB
-          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" +
+          $env:LIB = "C:\BuildTools\VC\Tools\MSVC\$latestVersion\lib\arm64;" + 
                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\arm64;" +
                     "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\arm64"
          Add-Content $env:GITHUB_ENV "LIB=$env:LIB"
@@ -266,7 +200,7 @@ jobs:
      - name: Install Rust
        run: |
          Invoke-WebRequest https://win.rustup.rs/x86_64 -OutFile rustup-init.exe
-          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc --default-toolchain 1.83.0
+          .\rustup-init.exe -y --default-host aarch64-pc-windows-msvc
        shell: powershell
      - name: Add Rust to PATH
        run: |
@@ -302,42 +236,5 @@ jobs:
      - name: Run tests
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --target aarch64-pc-windows-msvc --features remote --locked
-
-  msrv:
-    # Check the minimum supported Rust version
-    name: MSRV Check - Rust v${{ matrix.msrv }}
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
-    env:
-      # Need up-to-date compilers for kernels
-      CC: clang-18
-      CXX: clang++-18
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install ${{ matrix.msrv }}
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ matrix.msrv }}
-      - name: Downgrade  dependencies
-        # These packages have newer requirements for MSRV
-        run: |
-          cargo update -p aws-sdk-bedrockruntime --precise 1.64.0
-          cargo update -p aws-sdk-dynamodb --precise 1.55.0
-          cargo update -p aws-config --precise 1.5.10
-          cargo update -p aws-sdk-kms --precise 1.51.0
-          cargo update -p aws-sdk-s3 --precise 1.65.0
-          cargo update -p aws-sdk-sso --precise 1.50.0
-          cargo update -p aws-sdk-ssooidc --precise 1.51.0
-          cargo update -p aws-sdk-sts --precise 1.51.0
-          cargo update -p home --precise 0.5.9
-      - name: cargo +${{ matrix.msrv }} check
-        run: cargo check --workspace --tests --benches --all-features
+          cargo build --target aarch64-pc-windows-msvc
+          cargo test --target aarch64-pc-windows-msvc
--- a/.github/workflows/upload_wheel/action.yml
+++ b/.github/workflows/upload_wheel/action.yml
@@ -17,12 +17,11 @@ runs:
    run: |
      python -m pip install --upgrade pip
      pip install twine
-      python3 -m pip install --upgrade pkginfo
  - name: Choose repo
    shell: bash
    id: choose_repo
    run: |
-      if [[ ${{ github.ref }} == *beta* ]]; then
+      if [ ${{ github.ref }} == "*beta*" ]; then
        echo "repo=fury" >> $GITHUB_OUTPUT
      else
        echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -33,7 +32,7 @@ runs:
      FURY_TOKEN: ${{ inputs.fury_token }}
      PYPI_TOKEN: ${{ inputs.pypi_token }}
    run: |
-      if [[ ${{ steps.choose_repo.outputs.repo }} == fury ]]; then
+      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
        echo "Uploading $WHEEL to Fury"
        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ venv
 .vscode
 .zed
 rust/target
+rust/Cargo.lock

 site

@@ -41,3 +42,5 @@ dist
 target

 **/sccache.log
+
+Cargo.lock
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,27 +1,21 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
+-   repo: https://github.com/pre-commit/pre-commit-hooks
    rev: v3.2.0
    hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-  - repo: https://github.com/astral-sh/ruff-pre-commit
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/astral-sh/ruff-pre-commit
    # Ruff version.
-    rev: v0.9.9
+    rev: v0.2.2
    hooks:
-      - id: ruff
-  # - repo: https://github.com/RobertCraigie/pyright-python
-  #   rev: v1.1.395
-  #   hooks:
-  #     - id: pyright
-  #       args: ["--project", "python"]
-  #       additional_dependencies: [pyarrow-stubs]
-  - repo: local
-    hooks:
-      - id: local-biome-check
-        name: biome check
-        entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
-        language: system
-        types: [text]
-        files: "nodejs/.*"
-        exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
+    - id: ruff
+- repo: local
+  hooks:
+    - id: local-biome-check
+      name: biome check
+      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+      language: system
+      types: [text]
+      files: "nodejs/.*"
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,78 +0,0 @@
-# Contributing to LanceDB
-
-LanceDB is an open-source project and we welcome contributions from the community.
-This document outlines the process for contributing to LanceDB.
-
-## Reporting Issues
-
-If you encounter a bug or have a feature request, please open an issue on the
-[GitHub issue tracker](https://github.com/lancedb/lancedb).
-
-## Picking an issue
-
-We track issues on the GitHub issue tracker. If you are looking for something to
-work on, check the [good first issue](https://github.com/lancedb/lancedb/contribute) label. These issues are typically the best described and have the smallest scope.
-
-If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue.
-
-## Configuring Git
-
-First, fork the repository on GitHub, then clone your fork:
-
-```bash
-git clone https://github.com/<username>/lancedb.git
-cd lancedb
-```
-
-Then add the main repository as a remote:
-
-```bash
-git remote add upstream https://github.com/lancedb/lancedb.git
-git fetch upstream
-```
-
-## Setting up your development environment
-
-We have development environments for Python, Typescript, and Java. Each environment has its own setup instructions.
-
-* [Python](python/CONTRIBUTING.md)
-* [Typescript](nodejs/CONTRIBUTING.md)
-<!-- TODO: add Java contributing guide -->
-* [Documentation](docs/README.md)
-
-
-## Best practices for pull requests
-
-For the best chance of having your pull request accepted, please follow these guidelines:
-
-1. Unit test all bug fixes and new features. Your code will not be merged if it
-   doesn't have tests.
-1. If you change the public API, update the documentation in the `docs` directory.
-1. Aim to minimize the number of changes in each pull request. Keep to solving
-   one problem at a time, when possible.
-1. Before marking a pull request ready-for-review, do a self review of your code.
-   Is it clear why you are making the changes? Are the changes easy to understand?
-1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples:
-    * New feature: `feat: adding foo API`
-    * Bug fix: `fix: issue with foo API`
-    * Documentation change: `docs: adding foo API documentation`
-1. If your pull request is a work in progress, leave the pull request as a draft.
-   We will assume the pull request is ready for review when it is opened.
-1. When writing tests, test the error cases. Make sure they have understandable
-   error messages.
-
-## Project structure
-
-The core library is written in Rust. The Python, Typescript, and Java libraries
-are wrappers around the Rust library.
-
-* `src/lancedb`: Rust library source code
-* `python`: Python package source code
-* `nodejs`: Typescript package source code
-* `node`: **Deprecated** Typescript package source code
-* `java`: Java package source code
-* `docs`: Documentation source code
-
-## Release process
-
-For information on the release process, see: [release_process.md](release_process.md)
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,55 +18,43 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.78.0"
+rust-version = "1.80.0"                                                     # TODO: lower this once we upgrade Lance again.

 [workspace.dependencies]
-lance = { "version" = "=0.24.1", "features" = ["dynamodb"] }
-lance-io = { version = "=0.24.1" }
-lance-index = { version = "=0.24.1" }
-lance-linalg = { version = "=0.24.1" }
-lance-table = { version = "=0.24.1" }
-lance-testing = { version = "=0.24.1" }
-lance-datafusion = { version = "=0.24.1" }
-lance-encoding = { version = "=0.24.1" }
+lance = { "version" = "=0.20.0", "features" = [
+    "dynamodb",
+], git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-index = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-linalg = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-table = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-testing = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-datafusion = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-encoding = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
 # Note that this one does not include pyarrow
-arrow = { version = "54.1", optional = false }
-arrow-array = "54.1"
-arrow-data = "54.1"
-arrow-ipc = "54.1"
-arrow-ord = "54.1"
-arrow-schema = "54.1"
-arrow-arith = "54.1"
-arrow-cast = "54.1"
+arrow = { version = "52.2", optional = false }
+arrow-array = "52.2"
+arrow-data = "52.2"
+arrow-ipc = "52.2"
+arrow-ord = "52.2"
+arrow-schema = "52.2"
+arrow-arith = "52.2"
+arrow-cast = "52.2"
 async-trait = "0"
-datafusion = { version = "45.0", default-features = false }
-datafusion-catalog = "45.0"
-datafusion-common = { version = "45.0", default-features = false }
-datafusion-execution = "45.0"
-datafusion-expr = "45.0"
-datafusion-physical-plan = "45.0"
-env_logger = "0.11"
+chrono = "0.4.35"
+datafusion-common = "41.0"
+datafusion-physical-plan = "41.0"
+env_logger = "0.10"
 half = { "version" = "=2.4.1", default-features = false, features = [
    "num-traits",
 ] }
 futures = "0"
 log = "0.4"
-moka = { version = "0.12", features = ["future"] }
-object_store = "0.11.0"
+moka = { version = "0.11", features = ["future"] }
+object_store = "0.10.2"
 pin-project = "1.0.7"
-snafu = "0.8"
+snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
 rand = "0.8"
 regex = "1.10"
 lazy_static = "1"
-semver = "1.0.25"
-
-# Temporary pins to work around downstream issues
-# https://github.com/apache/arrow-rs/commit/2fddf85afcd20110ce783ed5b4cdeb82293da30b
-chrono = "=0.4.39"
-# https://github.com/RustCrypto/formats/issues/1684
-base64ct = "=1.6.0"
-
-# Workaround for: https://github.com/eira-fransham/crunchy/issues/13
-crunchy = "=0.2.2"
--- a/ci/build_linux_artifacts.sh
+++ b/ci/build_linux_artifacts.sh
@@ -1,9 +1,8 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}

-# We pass down the current user so that when we later mount the local files
+# We pass down the current user so that when we later mount the local files 
 # into the container, the files are accessible by the current user.
 pushd ci/manylinux_node
 docker build \
@@ -19,4 +18,4 @@ docker run \
    -v $(pwd):/io -w /io \
    --memory-swap=-1 \
    lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE
+    bash ci/manylinux_node/build_vectordb.sh $ARCH
--- a/ci/manylinux_node/build_vectordb.sh
+++ b/ci/manylinux_node/build_vectordb.sh
@@ -2,7 +2,6 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}

 if [ "$ARCH" = "x86_64" ]; then
    export OPENSSL_LIB_DIR=/usr/local/lib64/
@@ -18,4 +17,4 @@ FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 cd node
 npm ci
 npm run build-release
-npm run pack-build -- -t $TARGET_TRIPLE
+npm run pack-build
--- a/ci/parse_requirements.py
+++ b/ci/parse_requirements.py
@@ -1,41 +0,0 @@
-import argparse
-import toml
-
-
-def parse_dependencies(pyproject_path, extras=None):
-    with open(pyproject_path, "r") as file:
-        pyproject = toml.load(file)
-
-    dependencies = pyproject.get("project", {}).get("dependencies", [])
-    for dependency in dependencies:
-        print(dependency)
-
-    optional_dependencies = pyproject.get("project", {}).get(
-        "optional-dependencies", {}
-    )
-
-    if extras:
-        for extra in extras.split(","):
-            for dep in optional_dependencies.get(extra, []):
-                print(dep)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Generate requirements.txt from pyproject.toml"
-    )
-    parser.add_argument("path", type=str, help="Path to pyproject.toml")
-    parser.add_argument(
-        "--extras",
-        type=str,
-        help="Comma-separated list of extras to include",
-        default="",
-    )
-
-    args = parser.parse_args()
-
-    parse_dependencies(args.path, args.extras)
-
-
-if __name__ == "__main__":
-    main()
--- a/ci/sysroot-aarch64-pc-windows-msvc.sh
+++ b/ci/sysroot-aarch64-pc-windows-msvc.sh
@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-
-# dbghelp.lib fwpuclnt.lib arm64rt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7a332420d812f7c1d41da865ae5a7c52/windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/19de98ed4a79938d0045d19c047936b3/3e2f7be479e3679d700ce0782e4cc318.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/227f40682a88dc5fa0ccb9cadc9ad30af99ad1f1a75db63407587d079f60d035/Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20arm64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.ARM64.Desktop.vsix
-
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/include
-mkdir -p /usr/aarch64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/aarch64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/aarch64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# ARM intrinsics
-# original dir: MSVC/
-
-# '__n128x4' redefined in arm_neon.h
-# "arm64_neon.h" included from intrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp arm_neon.h intrin.h -t /usr/aarch64-pc-windows-msvc/usr/include)
-
-# .lib
-
-# _Interlocked intrinsics
-# must always link with arm64rt.lib
-# reason: https://developercommunity.visualstudio.com/t/libucrtlibstreamobj-error-lnk2001-unresolved-exter/1544787#T-ND1599818
-# I don't understand the 'correct' fix for this, arm64rt.lib is supposed to be the workaround
-
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/arm64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib runtimeobject.lib dbghelp.lib fwpuclnt.lib arm64rt.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/arm64' && cp libcmt.lib libvcruntime.lib -t /usr/aarch64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/arm64/libucrt.lib' /usr/aarch64-pc-windows-msvc/usr/lib
--- a/ci/sysroot-x86_64-pc-windows-msvc.sh
+++ b/ci/sysroot-x86_64-pc-windows-msvc.sh
@@ -1,105 +0,0 @@
-#!/bin/sh
-
-# https://github.com/mstorsjo/msvc-wine/blob/master/vsdownload.py
-# https://github.com/mozilla/gecko-dev/blob/6027d1d91f2d3204a3992633b3ef730ff005fc64/build/vs/vs2022-car.yaml
-
-# function dl() {
-# 	curl -O https://download.visualstudio.microsoft.com/download/pr/$1
-# }
-
-# [[.h]]
-
-# "id": "Win11SDK_10.0.26100"
-# "version": "10.0.26100.7"
-
-# libucrt.lib
-
-# example: <assert.h>
-# dir: ucrt/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ee3a5fc6e9fc832af7295b138e93839/universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b1aa09b90fe314aceb090f6ec7626624/16ab2ea2187acffa6435e334796c8c89.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/400609bb0ff5804e36dbe6dcd42a7f01/6ee7bbee8435130a869cf971694fd9e2.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/2ac327317abb865a0e3f56b2faefa918/78fa3c824c2c48bd4a49ab5969adaaf7.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/f034bc0b2680f67dccd4bfeea3d0f932/7afc7b670accd8e3cc94cfffd516f5cb.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/7ed5e12f9d50f80825a8b27838cf4c7f/96076045170fe5db6d5dcf14b6f6688e.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/764edc185a696bda9e07df8891dddbbb/a1e2a83aa8a71c48c742eeaff6e71928.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/66854bedc6dbd5ccb5dd82c8e2412231/b2f03f34ff83ec013b9e45c7cd8e8a73.cab
-
-# example: <windows.h>
-# dir: um/
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/b286efac4d83a54fc49190bddef1edc9/windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/e0dc3811d92ab96fcb72bf63d6c08d71/766c0ffd568bbb31bf7fb6793383e24a.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/613503da4b5628768497822826aed39f/8125ee239710f33ea485965f76fae646.cab
-
-# example: <winapifamily.h>
-# dir: /shared
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/122979f0348d3a2a36b6aa1a111d5d0c/windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/766e04beecdfccff39e91dd9eb32834a/e89e3dcbb016928c7e426238337d69eb.cab
-
-
-# "id": "Microsoft.VisualC.14.16.CRT.Headers"
-# "version": "14.16.27045"
-
-# example: <vcruntime.h>
-# dir: MSVC/
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/87bbe41e09a2f83711e72696f49681429327eb7a4b90618c35667a6ba2e2880e/Microsoft.VisualC.14.16.CRT.Headers.vsix
-
-# [[.lib]]
-
-# advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/944c4153b849a1f7d0c0404a4f1c05ea/windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5306aed3e1a38d1e8bef5934edeb2a9b/05047a45609f311645eebcac2739fc4c.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/13c8a73a0f5a6474040b26d016a26fab/13d68b8a7b6678a368e2d13ff4027521.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/149578fb3b621cdb61ee1813b9b3e791/463ad1b0783ebda908fd6c16a4abfe93.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/5c986c4f393c6b09d5aec3b539e9fb4a/5a22e5cde814b041749fb271547f4dd5.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/bfc3904a0195453419ae4dfea7abd6fb/e10768bb6e9d0ea730280336b697da66.cab
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/637f9f3be880c71f9e3ca07b4d67345c/f9b24c8280986c0683fbceca5326d806.cab
-
-# dbghelp.lib fwpuclnt.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/9f51690d5aa804b1340ce12d1ec80f89/windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-curl -O https://download.visualstudio.microsoft.com/download/pr/32863b8d-a46d-4231-8e84-0888519d20a9/d3a7df4ca3303a698640a29e558a5e5b/58314d0646d7e1a25e97c902166c3155.cab
-
-# libcmt.lib libvcruntime.lib
-curl -O https://download.visualstudio.microsoft.com/download/pr/bac0afd7-cc9e-4182-8a83-9898fa20e092/8728f21ae09940f1f4b4ee47b4a596be2509e2a47d2f0c83bbec0ea37d69644b/Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-
-msiextract universal%20crt%20headers%20libraries%20and%20sources-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20headers%20onecoreuap-x86_en-us.msi
-msiextract windows%20sdk%20for%20windows%20store%20apps%20libs-x86_en-us.msi
-msiextract windows%20sdk%20desktop%20libs%20x64-x86_en-us.msi
-unzip -o Microsoft.VisualC.14.16.CRT.Headers.vsix
-unzip -o Microsoft.VisualC.14.16.CRT.x64.Desktop.vsix
-
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/include
-mkdir -p /usr/x86_64-pc-windows-msvc/usr/lib
-
-# lowercase folder/file names
-echo "$(find . -regex ".*/[^/]*[A-Z][^/]*")" | xargs -I{} sh -c 'mv "$(echo "{}" | sed -E '"'"'s/(.*\/)/\L\1/'"'"')" "$(echo "{}" | tr [A-Z] [a-z])"'
-
-# .h
-(cd 'program files/windows kits/10/include/10.0.26100.0' && cp -r ucrt/* um/* shared/* -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-cp -r contents/vc/tools/msvc/14.16.27023/include/* /usr/x86_64-pc-windows-msvc/usr/include
-
-# lowercase #include "" and #include <>
-find /usr/x86_64-pc-windows-msvc/usr/include -type f -exec sed -i -E 's/(#include <[^<>]*?[A-Z][^<>]*?>)|(#include "[^"]*?[A-Z][^"]*?")/\L\1\2/' "{}" ';'
-
-# x86 intrinsics
-# original dir: MSVC/
-
-# '_mm_movemask_epi8' defined in emmintrin.h
-# '__v4sf' defined in xmmintrin.h
-# '__v2si' defined in mmintrin.h
-# '__m128d' redefined in immintrin.h
-# '__m128i' redefined in intrin.h
-# '_mm_comlt_epu8' defined in ammintrin.h
-
-(cd /usr/lib/llvm19/lib/clang/19/include && cp emmintrin.h xmmintrin.h mmintrin.h immintrin.h intrin.h ammintrin.h -t /usr/x86_64-pc-windows-msvc/usr/include)
-
-# .lib
-(cd 'program files/windows kits/10/lib/10.0.26100.0/um/x64' && cp advapi32.lib bcrypt.lib kernel32.lib ntdll.lib user32.lib uuid.lib ws2_32.lib userenv.lib cfgmgr32.lib dbghelp.lib fwpuclnt.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-(cd 'contents/vc/tools/msvc/14.16.27023/lib/x64' && cp libcmt.lib libvcruntime.lib -t /usr/x86_64-pc-windows-msvc/usr/lib)
-
-cp 'program files/windows kits/10/lib/10.0.26100.0/ucrt/x64/libucrt.lib' /usr/x86_64-pc-windows-msvc/usr/lib
--- a/ci/validate_stable_lance.py
+++ b/ci/validate_stable_lance.py
@@ -1,34 +0,0 @@
-import tomllib
-
-found_preview_lance = False
-
-with open("Cargo.toml", "rb") as f:
-    cargo_data = tomllib.load(f)
-
-    for name, dep in cargo_data["workspace"]["dependencies"].items():
-        if name == "lance" or name.startswith("lance-"):
-            if isinstance(dep, str):
-                version = dep
-            elif isinstance(dep, dict):
-                # Version doesn't have the beta tag in it, so we instead look
-                # at the git tag.
-                version = dep.get('tag', dep.get('version'))
-            else:
-                raise ValueError("Unexpected type for dependency: " + str(dep))
-
-            if "beta" in version:
-                found_preview_lance = True
-                print(f"Dependency '{name}' is a preview version: {version}")
-
-with open("python/pyproject.toml", "rb") as f:
-    py_proj_data = tomllib.load(f)
-
-    for dep in py_proj_data["project"]["dependencies"]:
-        if dep.startswith("pylance"):
-            if "b" in dep:
-                found_preview_lance = True
-                print(f"Dependency '{dep}' is a preview version")
-            break  # Only one pylance dependency
-
-if found_preview_lance:
-    raise ValueError("Found preview version of Lance in dependencies")
--- a/docs/README.md
+++ b/docs/README.md
@@ -9,81 +9,36 @@ unreleased features.
 ## Building the docs

 ### Setup
-1. Install LanceDB Python. See setup in [Python contributing guide](../python/CONTRIBUTING.md).
-   Run `make develop` to install the Python package.
-2. Install documentation dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed

-### Preview the docs
+### Building node module and create markdown files

-```shell
+See [Javascript docs README](./src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+### Run local server
+
+You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
+
+```bash
 cd docs
 mkdocs serve
 ```

-If you want to just generate the HTML files:
+### Run doctest for typescript example

-```shell
-PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
-```
-
-If successful, you should see a `docs/site` directory that you can verify locally.
-
-## Adding examples
-
-To make sure examples are correct, we put examples in test files so they can be
-run as part of our test suites.
-
-You can see the tests are at:
-
-* Python: `python/python/tests/docs`
-* Typescript: `nodejs/examples/`
-
-### Checking python examples
-
-```shell
-cd python
-pytest -vv python/tests/docs
-```
-
-### Checking typescript examples
-
-The `@lancedb/lancedb` package must be built before running the tests:
-
-```shell
-pushd nodejs
-npm ci
+```bash
+cd lancedb/docs
+npm i
 npm run build
-popd
-```
-
-Then you can run the examples by going to the `nodejs/examples` directory and
-running the tests like a normal npm package:
-
-```shell
-pushd nodejs/examples
-npm ci
-npm test
-popd
-```
-
-## API documentation
-
-### Python
-
-The Python API documentation is organized based on the file `docs/src/python/python.md`.
-We manually add entries there so we can control the organization of the reference page.
-**However, this means any new types must be manually added to the file.** No additional
-steps are needed to generate the API documentation.
-
-### Typescript
-
-The typescript API documentation is generated from the typescript source code using [typedoc](https://typedoc.org/).
-
-When new APIs are added, you must manually re-run the typedoc command to update the API documentation.
-The new files should be checked into the repository.
-
-```shell
-pushd nodejs
-npm run docs
-popd
+npm run all
 ```
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -4,9 +4,6 @@ repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
-watch:
-  - src
-  - ../python/python

 theme:
  name: "material"
@@ -58,15 +55,10 @@ plugins:
            show_signature_annotations: true
            show_root_heading: true
            members_order: source
-            docstring_section_style: list
-            signature_crossrefs: true
-            separate_signature: true
          import:
            # for cross references
            - https://arrow.apache.org/docs/objects.inv
            - https://pandas.pydata.org/docs/objects.inv
-            - https://lancedb.github.io/lance/objects.inv
-            - https://docs.pydantic.dev/latest/objects.inv
  - mkdocs-jupyter
  - render_swagger:
      allow_arbitrary_locations: true
@@ -109,8 +101,8 @@ nav:
      - 📚 Concepts:
          - Vector search: concepts/vector_search.md
          - Indexing:
-              - IVFPQ: concepts/index_ivfpq.md
-              - HNSW: concepts/index_hnsw.md
+            - IVFPQ: concepts/index_ivfpq.md
+            - HNSW: concepts/index_hnsw.md
          - Storage: concepts/storage.md
          - Data management: concepts/data_management.md
      - 🔨 Guides:
@@ -134,8 +126,8 @@ nav:
              - Adaptive RAG: rag/adaptive_rag.md
              - SFR RAG: rag/sfr_rag.md
              - Advanced Techniques:
-                  - HyDE: rag/advanced_techniques/hyde.md
-                  - FLARE: rag/advanced_techniques/flare.md
+                - HyDE: rag/advanced_techniques/hyde.md
+                - FLARE: rag/advanced_techniques/flare.md
          - Reranking:
              - Quickstart: reranking/index.md
              - Cohere Reranker: reranking/cohere.md
@@ -150,9 +142,7 @@ nav:
              - Building Custom Rerankers: reranking/custom_reranker.md
              - Example: notebooks/lancedb_reranking.ipynb
          - Filtering: sql.md
-          - Versioning & Reproducibility:
-              - sync API: notebooks/reproducibility.ipynb
-              - async API: notebooks/reproducibility_async.ipynb
+          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
          - Configuring Storage: guides/storage.md
          - Migration Guide: migration.md
          - Tuning retrieval performance:
@@ -182,7 +172,6 @@ nav:
                  - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
                  - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
          - User-defined embedding functions: embeddings/custom_embedding_function.md
-          - Variables and secrets: embeddings/variables_and_secrets.md
          - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
          - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
      - 🔌 Integrations:
@@ -239,14 +228,13 @@ nav:
              - 🐍 Python: python/saas-python.md
              - 👾 JavaScript: javascript/modules.md
              - REST API: cloud/rest.md
-          - FAQs: cloud/cloud_faq.md

  - Quick start: basic.md
  - Concepts:
      - Vector search: concepts/vector_search.md
      - Indexing:
-          - IVFPQ: concepts/index_ivfpq.md
-          - HNSW: concepts/index_hnsw.md
+        - IVFPQ: concepts/index_ivfpq.md
+        - HNSW: concepts/index_hnsw.md
      - Storage: concepts/storage.md
      - Data management: concepts/data_management.md
  - Guides:
@@ -270,8 +258,8 @@ nav:
          - Adaptive RAG: rag/adaptive_rag.md
          - SFR RAG: rag/sfr_rag.md
          - Advanced Techniques:
-              - HyDE: rag/advanced_techniques/hyde.md
-              - FLARE: rag/advanced_techniques/flare.md
+            - HyDE: rag/advanced_techniques/hyde.md
+            - FLARE: rag/advanced_techniques/flare.md
      - Reranking:
          - Quickstart: reranking/index.md
          - Cohere Reranker: reranking/cohere.md
@@ -285,9 +273,7 @@ nav:
          - Building Custom Rerankers: reranking/custom_reranker.md
          - Example: notebooks/lancedb_reranking.ipynb
      - Filtering: sql.md
-      - Versioning & Reproducibility:
-          - sync API: notebooks/reproducibility.ipynb
-          - async API: notebooks/reproducibility_async.ipynb
+      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
      - Configuring Storage: guides/storage.md
      - Migration Guide: migration.md
      - Tuning retrieval performance:
@@ -316,7 +302,6 @@ nav:
              - Imagebind embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/imagebind_embedding.md
              - Jina Embeddings: embeddings/available_embedding_models/multimodal_embedding_functions/jina_multimodal_embedding.md
      - User-defined embedding functions: embeddings/custom_embedding_function.md
-      - Variables and secrets: embeddings/variables_and_secrets.md
      - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
      - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
  - Integrations:
@@ -355,8 +340,8 @@ nav:
      - 🦀 Rust:
          - Overview: examples/examples_rust.md
  - Studies:
-      - studies/overview.md
-      - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
+    - studies/overview.md
+    - ↗Improve retrievers with hybrid search and reranking: https://blog.lancedb.com/hybrid-search-and-reranking-report/
  - API reference:
      - Overview: api_reference.md
      - Python: python/python.md
@@ -369,7 +354,6 @@ nav:
          - 🐍 Python: python/saas-python.md
          - 👾 JavaScript: javascript/modules.md
          - REST API: cloud/rest.md
-      - FAQs: cloud/cloud_faq.md

 extra_css:
  - styles/global.css
@@ -377,7 +361,6 @@ extra_css:

 extra_javascript:
  - "extra_js/init_ask_ai_widget.js"
-  - "extra_js/reo.js"

 extra:
  analytics:
--- a/docs/openapi.yml
+++ b/docs/openapi.yml
@@ -38,13 +38,6 @@ components:
      required: true
      schema:
        type: string
-    index_name:
-      name: index_name
-      in: path
-      description: name of the index
-      required: true
-      schema:
-        type: string
  responses:
    invalid_request:
      description: Invalid request
@@ -492,22 +485,3 @@ paths:
          $ref: "#/components/responses/unauthorized"
        "404":
          $ref: "#/components/responses/not_found"
-  /v1/table/{name}/index/{index_name}/drop/:
-    post:
-      description: Drop an index from the table
-      tags:
-        - Tables
-      summary: Drop an index from the table
-      operationId: dropIndex
-      parameters:
-        - $ref: "#/components/parameters/table_name"
-        - $ref: "#/components/parameters/index_name"
-      responses:
-        "200":
-          description: Index successfully dropped
-        "400":
-          $ref: "#/components/responses/invalid_request"
-        "401":
-          $ref: "#/components/responses/unauthorized"
-        "404":
-          $ref: "#/components/responses/not_found"
--- a/docs/src/ann_indexes.md
+++ b/docs/src/ann_indexes.md
@@ -18,24 +18,25 @@ See the [indexing](concepts/index_ivfpq.md) concepts guide for more information
 Lance supports `IVF_PQ` index type by default.

 === "Python"
-    === "Sync API"

-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index"
-        ```
-    === "Async API"
-        Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
+    ```python
+    import lancedb
+    import numpy as np
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-ivfpq"
-        --8<-- "python/python/tests/docs/test_guide_index.py:create_ann_index_async"
-        ```
+    # Create 10,000 sample vectors
+    data = [{"vector": row, "item": f"item {i}"}
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+    # Add the vectors to a table
+    tbl = db.create_table("my_vectors", data=data)
+
+    # Create and train the index - you need to have enough data in the table for an effective training step
+    tbl.create_index(num_partitions=256, num_sub_vectors=96)
+    ```

 === "TypeScript"

@@ -82,7 +83,6 @@ The following IVF_PQ paramters can be specified:
 - **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
  a single PQ code. The default is the dimension of the vector divided by 16.
- **num_bits**: The number of bits used to encode each sub-vector. Only 4 and 8 are supported. The higher the number of bits, the higher the accuracy of the index, also the slower search. The default is 8.

 !!! note

@@ -126,9 +126,7 @@ You can specify the GPU device to train IVF partitions via
        accelerator="mps"
    )
    ```
-!!! note
-    GPU based indexing is not yet supported with our asynchronous client.
-    
+
 Troubleshooting:

 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
@@ -144,25 +142,23 @@ There are a couple of parameters that can be used to fine-tune the search:
 - **nprobes** (default: 20): The number of probes used. A higher number makes search more accurate but also slower.<br/>
  Most of the time, setting nprobes to cover 5-15% of the dataset should achieve high recall with low latency.<br/>
    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, `nprobes` should be set to ~20-40. This value can be adjusted to achieve the optimal balance between search latency and search quality. <br/>
-
+  
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
    - _For example_, For a dataset of 1 million vectors divided into 256 partitions, setting the `refine_factor` to 200 will initially retrieve the top 4,000 candidates (top k * refine_factor) from all searched partitions. These candidates are then reranked to determine the final top 20 results.<br/>
-!!! note
+!!! note 
    Both `nprobes` and `refine_factor` are only applicable if an ANN index is present. If specified on a table without an ANN index, those parameters are ignored.


 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async"
-        ```
+    ```python
+    tbl.search(np.random.random((1536))) \
+        .limit(2) \
+        .nprobes(20) \
+        .refine_factor(10) \
+        .to_pandas()
+    ```

    ```text
                                              vector       item       _distance
@@ -199,16 +195,10 @@ The search will return the data requested in addition to the distance of each it
 You can further filter the elements returned by a search using a where clause.

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_filter"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_filter"
-        ```
+    ```python
+    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+    ```

 === "TypeScript"

@@ -230,16 +220,10 @@ You can select the columns returned by the query using a select clause.

 === "Python"

-    === "Sync API"
+    ```python
+    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+    ```

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_select"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_async_with_select"
-        ```

    ```text
                                                vector _distance
@@ -304,4 +288,4 @@ less space distortion, and thus yields better accuracy. However, a higher `num_s

 `m` determines the number of connections a new node establishes with its closest neighbors upon entering the graph. Typically, `m` falls within the range of 5 to 48. Lower `m` values are suitable for low-dimensional data or scenarios where recall is less critical. Conversely, higher `m` values are beneficial for high-dimensional data or when high recall is required. In essence, a larger `m` results in a denser graph with increased connectivity, but at the expense of higher memory consumption.

-`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase
+`ef_construction` balances build speed and accuracy. Higher values increase accuracy but slow down the build process. A typical range is 150 to 300. For good search results, a minimum value of 100 is recommended. In most cases, setting this value above 500 offers no additional benefit. Ensure that `ef_construction` is always set to a value equal to or greater than `ef` in the search phase
--- a/docs/src/ann_indexes.ts
+++ b/docs/src/ann_indexes.ts
@@ -3,7 +3,6 @@ import * as vectordb from "vectordb";
 // --8<-- [end:import]

 (async () => {
-  console.log("ann_indexes.ts: start");
  // --8<-- [start:ingest]
  const db = await vectordb.connect("data/sample-lancedb");

@@ -50,5 +49,5 @@ import * as vectordb from "vectordb";
    .execute();
  // --8<-- [end:search3]

-  console.log("ann_indexes.ts: done");
+  console.log("Ann indexes: done");
 })();
--- a/docs/src/assets/maxsim.png
+++ b/docs/src/assets/maxsim.png
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -133,22 +133,21 @@ recommend switching to stable releases.
 ## Connect to a database

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:imports"
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:imports"
+    --8<-- "python/python/tests/docs/test_basic.py:connect"

-        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-        --8<-- "python/python/tests/docs/test_basic.py:connect"
-        ```
-    === "Async API"
+    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
+    ```

-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:imports"
+    !!! note "Asynchronous Python API"

-        --8<-- "python/python/tests/docs/test_basic.py:set_uri"
-        --8<-- "python/python/tests/docs/test_basic.py:connect_async"
-        ```
+        The asynchronous Python API is new and has some slight differences compared
+        to the synchronous API.  Feel free to start using the asynchronous version.
+        Once all features have migrated we will start to move the synchronous API to
+        use the same syntax as the asynchronous API.  To help with this migration we
+        have created a [migration guide](migration.md) detailing the differences.

 === "Typescript[^1]"

@@ -192,33 +191,21 @@ table.

 === "Python"

+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:create_table"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
+    ```
+
    If the table already exists, LanceDB will raise an error by default.
    If you want to overwrite the table, you can pass in `mode="overwrite"`
    to the `create_table` method.

-    === "Sync API"
+    You can also pass in a pandas DataFrame directly:

-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table"
-        ```
-
-        You can also pass in a pandas DataFrame directly:
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
-        ```
-
-        You can also pass in a pandas DataFrame directly:
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
+    ```

 === "Typescript[^1]"

@@ -268,16 +255,10 @@ similar to a `CREATE TABLE` statement in SQL.

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
-        ```
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+      ```

    !!! note "You can define schema in Pydantic"
        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
@@ -308,16 +289,10 @@ Once created, you can open a table as follows:

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:open_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:open_table"
+    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
+    ```

 === "Typescript[^1]"
    === "@lancedb/lancedb"
@@ -343,16 +318,10 @@ If you forget the name of your table, you can always get a listing of all table

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:table_names"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:table_names"
+    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
+    ```

 === "Typescript[^1]"
    === "@lancedb/lancedb"
@@ -379,16 +348,10 @@ After a table has been created, you can always add more data to it as follows:

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_data"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:add_data"
+    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+    ```

 === "Typescript[^1]"
    === "@lancedb/lancedb"
@@ -415,16 +378,10 @@ Once you've embedded the query, you can find its nearest neighbors as follows:

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:vector_search"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
+    ```

    This returns a pandas DataFrame with the results.

@@ -463,16 +420,10 @@ LanceDB allows you to create an ANN index on a table as follows:

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
-        ```
+    ```py
+    --8<-- "python/python/tests/docs/test_basic.py:create_index"
+    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
+    ```

 === "Typescript[^1]"
    === "@lancedb/lancedb"
@@ -508,16 +459,10 @@ This can delete any number of rows that match the filter.

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
+    ```

 === "Typescript[^1]"

@@ -546,10 +491,7 @@ simple or complex as needed. To see what expressions are supported, see the

 === "Python"

-    === "Sync API"
-        Read more: [lancedb.table.Table.delete][]
-    === "Async API"
-        Read more: [lancedb.table.AsyncTable.delete][]
+      Read more: [lancedb.table.Table.delete][]

 === "Typescript[^1]"

@@ -571,16 +513,10 @@ Use the `drop_table()` method on the database to remove a table.

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-        ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+    ```

    This permanently removes the table and is not recoverable, unlike deleting rows.
    By default, if the table does not exist an exception is raised. To suppress this,
@@ -615,17 +551,10 @@ You can use the embedding API when working with embedding models. It automatical

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
-
-        --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
-        ```
-    === "Async API"
-
-        Coming soon to the async API.
-        https://github.com/lancedb/lancedb/issues/1938
+    ```python
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
+    ```

 === "Typescript[^1]"

--- a/docs/src/basic_legacy.ts
+++ b/docs/src/basic_legacy.ts
@@ -107,6 +107,7 @@ const example = async () => {
  // --8<-- [start:search]
  const query = await tbl.search([100, 100]).limit(2).execute();
  // --8<-- [end:search]
+  console.log(query);

  // --8<-- [start:delete]
  await tbl.delete('item = "fizz"');
@@ -118,9 +119,8 @@ const example = async () => {
 };

 async function main() {
-  console.log("basic_legacy.ts: start");
  await example();
-  console.log("basic_legacy.ts: done");
+  console.log("Basic example: done");
 }

 main();
--- a/docs/src/cloud/cloud_faq.md
+++ b/docs/src/cloud/cloud_faq.md
@@ -1,34 +0,0 @@
-This section provides answers to the most common questions asked about LanceDB Cloud. By following these guidelines, you can ensure a smooth, performant experience with LanceDB Cloud.
-
-### Should I reuse the database connection?
-Yes! It is recommended to establish a single database connection and maintain it throughout your interaction with the tables within. 
-
-LanceDB uses HTTP connections to communicate with the servers. By re-using the Connection object, you avoid the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
-
-### Should I re-use the `Table` object?
-`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the **latest version** of the data. 
-
-### What should I do if I need to search for rows by `id`?
-LanceDB Cloud currently does not support an ID or primary key column. You are recommended to add a 
-user-defined ID column. To significantly improve the query performance with SQL causes, a scalar BITMAP/BTREE index should be created on this column. 
-
-### What are the vector indexing types supported by LanceDB Cloud?
-We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff between query latency and query quality.
-
-### When I add new rows to a table, do I need to manually update the index?
-No! LanceDB Cloud triggers an asynchronous background job to index the new vectors.
-
-Even though indexing is asynchronous, your vectors will still be immediately searchable. LanceDB uses brute-force search to search over unindexed rows. This makes you new data is immediately available, but does increase latency temporarily. To disable the brute-force part of search, set the `fast_search` flag in your query to `true`.
-
-### Do I need to reindex the whole dataset if only a small portion of the data is deleted or updated?
-No! Similar to adding data to the table, LanceDB Cloud triggers an asynchronous background job to update the existing indices. Therefore, no action is needed from users and there is absolutely no 
-downtime expected.
-
-### How do I know whether an index has been created?
-While index creation in LanceDB Cloud is generally fast, querying immediately after a `create_index` call may result in errors. It's recommended to use `list_indices` to verify index creation before querying.
-
-### Why is my query latency higher than expected?
-Multiple factors can impact query latency. To reduce query latency, consider the following:
- Send pre-warm queries: send a few queries to warm up the cache before an actual user query.
- Check network latency: LanceDB Cloud is hosted in AWS `us-east-1` region. It is recommended to run queries from an EC2 instance that is in the same region.
- Create scalar indices: If you are filtering on metadata, it is recommended to create scalar indices on those columns. This will speedup searches with metadata filtering. See [here](../guides/scalar_index.md) for more details on creating a scalar index.
--- a/docs/src/concepts/index_hnsw.md
+++ b/docs/src/concepts/index_hnsw.md
@@ -7,7 +7,7 @@ Approximate Nearest Neighbor (ANN) search is a method for finding data points ne
 There are three main types of ANN search algorithms:

 * **Tree-based search algorithms**: Use a tree structure to organize and store data points.
-* **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
+* * **Hash-based search algorithms**: Use a specialized geometric hash table to store and manage data points. These algorithms typically focus on theoretical guarantees, and don't usually perform as well as the other approaches in practice.
 * **Graph-based search algorithms**: Use a graph structure to store data points, which can be a bit complex. 

 HNSW is a graph-based algorithm. All graph-based search algorithms rely on the idea of a k-nearest neighbor (or k-approximate nearest neighbor) graph, which we outline below.  
--- a/docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
+++ b/docs/src/embeddings/available_embedding_models/text_embedding_functions/openai_embedding.md
@@ -6,7 +6,6 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
 | `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
-| `use_azure` | bool | `False` | Set true to use Azure OpenAPI SDK |


 ```python
--- a/docs/src/embeddings/custom_embedding_function.md
+++ b/docs/src/embeddings/custom_embedding_function.md
@@ -55,14 +55,6 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp

 This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.

-!!! danger "Use sensitive keys to prevent leaking secrets"
-    To prevent leaking secrets, such as API keys, you should add any sensitive
-    parameters of an embedding function to the output of the
-    [sensitive_keys()][lancedb.embeddings.base.EmbeddingFunction.sensitive_keys] /
-    [getSensitiveKeys()](../../js/namespaces/embedding/classes/EmbeddingFunction/#getsensitivekeys)
-    method. This prevents users from accidentally instantiating the embedding
-    function with hard-coded secrets.
-
 Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.

 === "Python"
--- a/docs/src/embeddings/variables_and_secrets.md
+++ b/docs/src/embeddings/variables_and_secrets.md
@@ -1,53 +0,0 @@
-# Variable and Secrets
-
-Most embedding configuration options are saved in the table's metadata. However,
-this isn't always appropriate. For example, API keys should never be stored in the
-metadata. Additionally, other configuration options might be best set at runtime,
-such as the `device` configuration that controls whether to use GPU or CPU for
-inference. If you hardcoded this to GPU, you wouldn't be able to run the code on
-a server without one.
-
-To handle these cases, you can set variables on the embedding registry and
-reference them in the embedding configuration. These variables will be available
-during the runtime of your program, but not saved in the table's metadata. When
-the table is loaded from a different process, the variables must be set again.
-
-To set a variable, use the `set_var()` / `setVar()` method on the embedding registry.
-To reference a variable, use the syntax `$env:VARIABLE_NAME`. If there is a default
-value, you can use the syntax `$env:VARIABLE_NAME:DEFAULT_VALUE`.
-
-## Using variables to set secrets
-
-Sensitive configuration, such as API keys, must either be set as environment
-variables or using variables on the embedding registry. If you pass in a hardcoded
-value, LanceDB will raise an error. Instead, if you want to set an API key via
-configuration, use a variable:
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:register_secret"
-    ```
-
-=== "Typescript"
-
-    ```typescript
-    --8<-- "nodejs/examples/embedding.test.ts:register_secret"
-    ```
-
-## Using variables to set the device parameter
-
-Many embedding functions that run locally have a `device` parameter that controls
-whether to use GPU or CPU for inference. Because not all computers have a GPU,
-it's helpful to be able to set the `device` parameter at runtime, rather than
-have it hard coded in the embedding configuration. To make it work even if the
-variable isn't set, you could provide a default value of `cpu` in the embedding
-configuration.
-
-Some embedding libraries even have a method to detect which devices are available,
-which could be used to dynamically set the device at runtime. For example, in Python
-you can check if a CUDA GPU is available using `torch.cuda.is_available()`.
-
-```python
--8<-- "python/python/tests/docs/test_embeddings_optional.py:register_device"
-```
--- a/docs/src/extra_js/reo.js
+++ b/docs/src/extra_js/reo.js
@@ -1 +0,0 @@
-!function(){var e,t,n;e="9627b71b382d201",t=function(){Reo.init({clientID:"9627b71b382d201"})},(n=document.createElement("script")).src="https://static.reo.dev/"+e+"/reo.js",n.defer=!0,n.onload=t,document.head.appendChild(n)}();
--- a/docs/src/fts.md
+++ b/docs/src/fts.md
@@ -10,20 +10,28 @@ LanceDB provides support for full-text search via Lance, allowing you to incorpo
 Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search, the FTS index must be created before you can search via keywords.

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts"
-        ```
-    === "Async API"
+    ```python
+    import lancedb

-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-        --8<-- "python/python/tests/docs/test_search.py:basic_fts_async"
-        ```
+    uri = "data/sample-lancedb"
+    db = lancedb.connect(uri)
+
+    table = db.create_table(
+        "my_table",
+        data=[
+            {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+            {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+        ],
+    )
+
+    # passing `use_tantivy=False` to use lance FTS index
+    # `use_tantivy=True` by default
+    table.create_fts_index("text", use_tantivy=False)
+    table.search("puppy").limit(10).select(["text"]).to_list()
+    # [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
+    # ...
+    ```

 === "TypeScript"

@@ -42,7 +50,7 @@ Consider that we have a LanceDB table named `my_table`, whose string column `tex
    });

    await tbl
-        .search("puppy", "fts")
+        .search("puppy", queryType="fts")
        .select(["text"])
        .limit(10)
        .toArray();
@@ -85,32 +93,22 @@ By default the text is tokenized by splitting on punctuation and whitespaces, an
 Stemming is useful for improving search results by reducing words to their root form, e.g. "running" to "run". LanceDB supports stemming for multiple languages, you can specify the tokenizer name to enable stemming by the pattern `tokenizer_name="{language_code}_stem"`, e.g. `en_stem` for English.

 For example, to enable stemming for English:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_stem_async"
-    ```
+```python
+table.create_fts_index("text", use_tantivy=True, tokenizer_name="en_stem")
+```

 the following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.

 The tokenizer is customizable, you can specify how the tokenizer splits the text, and how it filters out words, etc.

 For example, for language with accents, you can specify the tokenizer to use `ascii_folding` to remove accents, e.g. 'é' to 'e':
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_config_folding_async"
-    ```
+```python
+table.create_fts_index("text",
+                        use_tantivy=False,
+                        language="French",
+                        stem=True,
+                        ascii_folding=True)
+```

 ## Filtering

@@ -121,16 +119,9 @@ This can be invoked via the familiar `where` syntax.
 With pre-filtering:
 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_prefiltering_async"
-        ```
+    ```python
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=True).to_list()
+    ```

 === "TypeScript"

@@ -160,16 +151,9 @@ With pre-filtering:
 With post-filtering:
 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_postfiltering_async"
-        ```
+    ```python
+    table.search("puppy").limit(10).where("meta='foo'", prefilte=False).to_list()
+    ```

 === "TypeScript"

@@ -207,16 +191,9 @@ or a **terms** search query like `old man sea`. For more details on the terms
 query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).

 To search for a phrase, the index must be created with `with_position=True`:
-=== "Sync API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:fts_with_position_async"
-    ```
+```python
+table.create_fts_index("text", use_tantivy=False, with_position=True)
+```
 This will allow you to search for phrases, but it will also significantly increase the index size and indexing time.


@@ -228,16 +205,10 @@ This can make the query more efficient, especially when the table is large and t

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_search.py:fts_incremental_index_async"
-        ```
+    ```python
+    table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
+    table.optimize()
+    ```

 === "TypeScript"

--- a/docs/src/fts_tantivy.md
+++ b/docs/src/fts_tantivy.md
@@ -2,7 +2,7 @@

 LanceDB also provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.

-The tantivy-based FTS is only available in Python synchronous APIs and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).
+The tantivy-based FTS is only available in Python and does not support building indexes on object storage or incremental indexing. If you need these features, try native FTS [native FTS](fts.md).

 ## Installation

--- a/docs/src/guides/scalar_index.md
+++ b/docs/src/guides/scalar_index.md
@@ -1,51 +1,38 @@
-# Building a Scalar Index
+# Building Scalar Index

-Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
-
-Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
+Similar to many SQL databases, LanceDB supports several types of Scalar indices to accelerate search
 over scalar columns.

- `BTREE`: The most common type is BTREE. The index stores a copy of the
-  column in sorted order. This sorted copy allows a binary search to be used to
-  satisfy queries.
- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
-  uses a series of bits to indicate whether a value is present in a row of a table
- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
-  support queries with `array_contains_all` and `array_contains_any`
-  using an underlying bitmap index.
+- `BTREE`: The most common type is BTREE. This index is inspired by the btree data structure
+  although only the first few layers of the btree are cached in memory.
+  It will perform well on columns with a large number of unique values and few rows per value.
+- `BITMAP`: this index stores a bitmap for each unique value in the column.
+  This index is useful for columns with a finite number of unique values and many rows per value.
+  For example, columns that represent "categories", "labels", or "tags"
+- `LABEL_LIST`: a special index that is used to index list columns whose values have a finite set of possibilities.
  For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.

-!!! tips "How to choose the right scalar index type"
-
-    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
-    
-    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
-    
-    `LABEL_LIST`: This index should be used for columns containing list-type data.
-
 | Data Type                                                       | Filter                                    | Index Type   |
 | --------------------------------------------------------------- | ----------------------------------------- | ------------ |
 | Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
 | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
 | List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |

-### Create a scalar index
 === "Python"

-    === "Sync API"
+    ```python
+    import lancedb
+    books = [
+      {"book_id": 1, "publisher": "plenty of books", "tags": ["fantasy", "adventure"]},
+      {"book_id": 2, "publisher": "book town", "tags": ["non-fiction"]},
+      {"book_id": 3, "publisher": "oreilly", "tags": ["textbook"]}
+    ]

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb-btree-bitmap"
-        --8<-- "python/python/tests/docs/test_guide_index.py:basic_scalar_index_async"
-        ```
+    db = lancedb.connect("./db")
+    table = db.create_table("books", books)
+    table.create_scalar_index("book_id")  # BTree by default
+    table.create_scalar_index("publisher", index_type="BITMAP")
+    ```

 === "Typescript"

@@ -59,22 +46,16 @@ over scalar columns.
        await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
        ```

-The following scan will be faster if the column `book_id` has a scalar index:
+For example, the following scan will be faster if the column `my_col` has a scalar index:

 === "Python"

-    === "Sync API"
+    ```python
+    import lancedb

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:search_with_scalar_index_async"
-        ```
+    table = db.open_table("books")
+    my_df = table.search().where("book_id = 2").to_pandas()
+    ```

 === "Typescript"

@@ -95,18 +76,22 @@ Scalar indices can also speed up scans containing a vector search or full text s

 === "Python"

-    === "Sync API"
+    ```python
+    import lancedb

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index"
-        ```
-    === "Async API"
+    data = [
+      {"book_id": 1, "vector": [1, 2]},
+      {"book_id": 2, "vector": [3, 4]},
+      {"book_id": 3, "vector": [5, 6]}
+    ]
+    table = db.create_table("book_with_embeddings", data)

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_index.py:vector_search_with_scalar_index_async"
-        ```
+    (
+        table.search([1, 2])
+        .where("book_id != 3", prefilter=True)
+        .to_pandas()
+    )
+    ```

 === "Typescript"

@@ -121,36 +106,3 @@ Scalar indices can also speed up scans containing a vector search or full text s
          .limit(10)
          .toArray();
        ```
-### Update a scalar index
-Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_index.py:update_scalar_index_async"
-        ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```
-
-!!! note
-
-    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.
--- a/docs/src/guides/storage.md
+++ b/docs/src/guides/storage.md
@@ -12,52 +12,25 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
 === "Python"

    AWS S3:
-    === "Sync API"

-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("s3://bucket/path")
+    ```

    Google Cloud Storage:

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("gs://bucket/path")
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("gs://bucket/path")
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("gs://bucket/path")
+    ```

    Azure Blob Storage:

-    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("az://bucket/path")
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("az://bucket/path")
-        ```
-    Note that for Azure, storage credentials must be configured. See [below](#azure-blob-storage) for more details.
-
+    ```python
+    import lancedb
+    db = lancedb.connect("az://bucket/path")
+    ```

 === "TypeScript"

@@ -114,28 +87,22 @@ In most cases, when running in the respective cloud and permissions are set up c
 export TIMEOUT=60s
 ```

+!!! note "`storage_options` availability"
+
+    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
+    It is not yet supported in the Python synchronous API.
+
 If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:

 === "Python"

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={"timeout": "60s"}
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"}
+    )
+    ```

 === "TypeScript"

@@ -163,29 +130,15 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
 === "Python"

    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect("s3://bucket/path")
-        table = db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async("s3://bucket/path")
-        async_table = await async_db.create_table(
-            "table",
-            [{"a": 1, "b": 2}],
-            storage_options={"timeout": "60s"}
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async("s3://bucket/path")
+    table = await db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"}
+    )
+    ```

 === "TypeScript"

@@ -243,32 +196,17 @@ These can be set as environment variables or passed in the `storage_options` par

 === "Python"

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "aws_access_key_id": "my-access-key",
-                "aws_secret_access_key": "my-secret-key",
-                "aws_session_token": "my-session-token",
-            }
-        )
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "aws_access_key_id": "my-access-key",
+            "aws_secret_access_key": "my-secret-key",
+            "aws_session_token": "my-session-token",
+        }
+    )
+    ```

 === "TypeScript"

@@ -412,22 +350,12 @@ name of the table to use.

 === "Python"

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```

 === "JavaScript"

@@ -515,30 +443,16 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m

 === "Python"

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://bucket/path",
-            storage_options={
-                "region": "us-east-1",
-                "endpoint": "http://minio:9000",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        }
+    )
+    ```

 === "TypeScript"

@@ -590,30 +504,16 @@ To configure LanceDB to use an S3 Express endpoint, you must set the storage opt

 === "Python"

-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )
-        ```
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "s3://my-bucket--use1-az4--x-s3/path",
-            storage_options={
-                "region": "us-east-1",
-                "s3_express": "true",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        }
+    )
+    ```

 === "TypeScript"

@@ -654,29 +554,15 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
 === "Python"

    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "gs://my-bucket/my-database",
-            storage_options={
-                "service_account": "path/to/service-account.json",
-            }
-        )    
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        }
+    )
+    ```

 === "TypeScript"

@@ -728,31 +614,16 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
 === "Python"

    <!-- skip-test -->
-    === "Sync API"
-
-        ```python
-        import lancedb
-        db = lancedb.connect(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )
-        ```
-    <!-- skip-test -->
-    === "Async API"
-
-        ```python
-        import lancedb
-        async_db = await lancedb.connect_async(
-            "az://my-container/my-database",
-            storage_options={
-                account_name: "some-account",
-                account_key: "some-key",
-            }
-        )   
-        ```
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "az://my-container/my-database",
+        storage_options={
+            account_name: "some-account",
+            account_key: "some-key",
+        }
+    )
+    ```

 === "TypeScript"

--- a/docs/src/guides/tables.md
+++ b/docs/src/guides/tables.md
@@ -12,18 +12,10 @@ Initialize a LanceDB connection and create a table

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:connect"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:connect_async"
-        ```
+    ```python
+    import lancedb
+    db = lancedb.connect("./.lancedb")
+    ```

    LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.

@@ -55,16 +47,18 @@ Initialize a LanceDB connection and create a table

 === "Python"

-    === "Sync API"
+    ```python
+    import lancedb

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table"
-        ```
-    === "Async API"
+    db = lancedb.connect("./.lancedb")

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async"
-        ```
+    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
+
+    db.create_table("my_table", data)
+
+    db["my_table"].head()
+    ```

    !!! info "Note"
        If the table already exists, LanceDB will raise an error by default.
@@ -73,30 +67,16 @@ Initialize a LanceDB connection and create a table
        and the table exists, then it simply opens the existing table. The data you
        passed in will NOT be appended to the table in that case.

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_exist_ok"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_exist_ok"
-        ```
+    ```python
+    db.create_table("name", data, exist_ok=True)
+    ```

    Sometimes you want to make sure that you start fresh. If you want to
    overwrite the table, you can pass in mode="overwrite" to the createTable function.

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_overwrite"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_overwrite"
-        ```
+    ```python
+    db.create_table("name", data, mode="overwrite")
+    ```

 === "Typescript[^1]"
    You can create a LanceDB table in JavaScript using an array of records as follows.
@@ -166,37 +146,34 @@ Initialize a LanceDB connection and create a table

 ### From a Pandas DataFrame

+```python
+import pandas as pd

-=== "Sync API"
+data = pd.DataFrame({
+    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+    "lat": [45.5, 40.1],
+    "long": [-122.7, -74.1]
+})

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pandas"
-    ```
-=== "Async API"
+db.create_table("my_table", data)

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pandas"
-    ```
+db["my_table"].head()
+```

 !!! info "Note"
    Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.

 The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.

-=== "Sync API"
+```python
+custom_schema = pa.schema([
+pa.field("vector", pa.list_(pa.float32(), 4)),
+pa.field("lat", pa.float32()),
+pa.field("long", pa.float32())
+])

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_custom_schema"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_custom_schema"
-    ```
+table = db.create_table("my_table", data, schema=custom_schema)
+```

 ### From a Polars DataFrame

@@ -205,38 +182,45 @@ written in Rust. Just like in Pandas, the Polars integration is enabled by PyArr
 under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
 is on the way.

-=== "Sync API"
+```python
+import polars as pl

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_polars"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_polars"
-    ```
+data = pl.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pl_table", data=data)
+```

 ### From an Arrow Table
 You can also create LanceDB tables directly from Arrow tables.
 LanceDB supports float16 data type!

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_arrow_table"
-        ```
-    === "Async API"
+    ```python
+    import pyarrows as pa
+    import numpy as np

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-polars"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-numpy"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_arrow_table"
-        ```
+    dim = 16
+    total = 2
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float16(), dim)),
+            pa.field("text", pa.string())
+        ]
+    )
+    data = pa.Table.from_arrays(
+        [
+            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
+                    pa.list_(pa.float16(), dim)),
+            pa.array(["foo", "bar"])
+        ],
+        ["vector", "text"],
+    )
+    tbl = db.create_table("f16_tbl", data, schema=schema)
+    ```

 === "Typescript[^1]"

@@ -266,22 +250,25 @@ can be configured with the vector dimensions. It is also important to note that
 LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
 (which itself derives from `pydantic.BaseModel`).

-=== "Sync API"
+```python
+from lancedb.pydantic import Vector, LanceModel

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_pydantic"
-    ```
-=== "Async API"
+class Content(LanceModel):
+    movie_id: int
+    vector: Vector(128)
+    genres: str
+    title: str
+    imdb_id: int

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-Content"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_pydantic"
-    ```
+    @property
+    def imdb_url(self) -> str:
+        return f"https://www.imdb.com/title/tt{self.imdb_id}"
+
+import pyarrow as pa
+db = lancedb.connect("~/.lancedb")
+table_name = "movielens_small"
+table = db.create_table(table_name, schema=Content)
+```

 #### Nested schemas

@@ -290,24 +277,22 @@ For example, you may want to store the document string
 and the document source name as a nested Document object:

 ```python
--8<-- "python/python/tests/docs/test_guide_tables.py:import-pydantic-basemodel"
--8<-- "python/python/tests/docs/test_guide_tables.py:class-Document"
+class Document(BaseModel):
+    content: str
+    source: str
 ```

 This can be used as the type of a LanceDB table column:

-=== "Sync API"
+```python
+class NestedSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    document: Document

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_nested_schema"
-    ```
-=== "Async API"
+tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+```

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:class-NestedSchema"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_nested_schema"
-    ```
 This creates a struct column called "document" that has two subfields
 called "content" and "source":

@@ -371,20 +356,29 @@ LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generat

 Here's an example using using `RecordBatch` iterator for creating tables.

-=== "Sync API"
+```python
+import pyarrow as pa

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_from_batch"
-    ```
-=== "Async API"
+def make_batches():
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
+                        pa.list_(pa.float32(), 4)),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"],
+        )

-    ```python
-    --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches"
-    --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_from_batch"
-    ```
+schema = pa.schema([
+    pa.field("vector", pa.list_(pa.float32(), 4)),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+db.create_table("batched_tale", make_batches(), schema=schema)
+```

 You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.

@@ -393,29 +387,15 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 === "Python"
    If you forget the name of your table, you can always get a listing of all table names.

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:list_tables_async"
-        ```
+    ```python
+    print(db.table_names())
+    ```

    Then, you can open any existing tables.

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:open_table_async"
-        ```
+    ```python
+    tbl = db.open_table("my_table")
+    ```

 === "Typescript[^1]"

@@ -438,41 +418,35 @@ You can create an empty table for scenarios where you want to add data to the ta


    An empty table can be initialized via a PyArrow schema.
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table"
-        ```
-    === "Async API"
+    ```python
+    import lancedb
+    import pyarrow as pa

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async"
-        ```
+    schema = pa.schema(
+      [
+          pa.field("vector", pa.list_(pa.float32(), 2)),
+          pa.field("item", pa.string()),
+          pa.field("price", pa.float32()),
+      ])
+    tbl = db.create_table("empty_table_add", schema=schema)
+    ```

    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
    that has been extended to support LanceDB specific types like `Vector`.

-    === "Sync API"
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, vector

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_pydantic"
-        ```
-    === "Async API"
+    class Item(LanceModel):
+        vector: Vector(2)
+        item: str
+        price: float

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:class-Item"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:create_empty_table_async_pydantic"
-        ```
+    tbl = db.create_table("empty_table_add", schema=Item.to_arrow_schema())
+    ```

    Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.

@@ -499,96 +473,86 @@ After a table has been created, you can always add more data to it using the `ad

    ### Add a Pandas DataFrame

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pandas"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pandas"
-        ```
+    ```python
+    df = pd.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
+    })
+    tbl.add(df)
+    ```

    ### Add a Polars DataFrame

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_polars"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_polars"
-        ```
+    ```python
+    df = pl.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
+    })
+    tbl.add(df)
+    ```

    ### Add an Iterator

    You can also add a large dataset batch in one go using Iterator of any supported data types.

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_batch"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:make_batches_for_add"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_batch"
-        ```
+    ```python
+    def make_batches():
+        for i in range(5):
+            yield [
+                    {"vector": [3.1, 4.1], "item": "peach", "price": 6.0},
+                    {"vector": [5.9, 26.5], "item": "pear", "price": 5.0}
+                ]
+    tbl.add(make_batches())
+    ```

    ### Add a PyArrow table

    If you have data coming in as a PyArrow table, you can add it directly to the LanceDB table.

-    === "Sync API"
+    ```python
+    pa_table = pa.Table.from_arrays(
+            [
+                pa.array([[9.1, 6.7], [9.9, 31.2]],
+                        pa.list_(pa.float32(), 2)),
+                pa.array(["mango", "orange"]),
+                pa.array([7.0, 4.0]),
+            ],
+            ["vector", "item", "price"],
+        )

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pyarrow"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pyarrow"
-        ```
+    tbl.add(pa_table)
+    ```

    ### Add a Pydantic Model

    Assuming that a table has been created with the correct schema as shown [above](#creating-empty-table), you can add data items that are valid Pydantic models to the table.

-    === "Sync API"
+    ```python
+    pydantic_model_items = [
+        Item(vector=[8.1, 4.7], item="pineapple", price=10.0),
+        Item(vector=[6.9, 9.3], item="avocado", price=9.0)
+    ]

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_from_pydantic"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:add_table_async_from_pydantic"
-        ```
+    tbl.add(pydantic_model_items)
+    ```

    ??? "Ingesting Pydantic models with LanceDB embedding API"
        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` field as None to allow LanceDB to automatically vectorize the data.

-        === "Sync API"
+        ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import get_registry

-            ```python
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_with_embedding"
-            ```
-        === "Async API"
+        db = lancedb.connect("~/tmp")
+        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")

-            ```python
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb-pydantic"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:import-embeddings"
-            --8<-- "python/python/tests/docs/test_guide_tables.py:create_table_async_with_embedding"
-            ```
+        class Schema(LanceModel):
+            text: str = embed_fcn.SourceField()
+            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
+
+        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
+        models = [Schema(text="hello"), Schema(text="world")]
+        tbl.add(models)
+        ```

 === "Typescript[^1]"

@@ -601,79 +565,50 @@ After a table has been created, you can always add more data to it using the `ad
    )
    ```

-## Upserting into a table
-
-Upserting lets you insert new rows or update existing rows in a table. To upsert
-in LanceDB, use the merge insert API.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic"
-        ```
-        **API Reference**: [lancedb.table.Table.merge_insert][]
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic_async"
-        ```
-        **API Reference**: [lancedb.table.AsyncTable.merge_insert][]
-
-=== "Typescript[^1]"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:upsert_basic"
-        ```
-        **API Reference**: [lancedb.Table.mergeInsert](../js/classes/Table.md/#mergeInsert)
-
-Read more in the guide on [merge insert](tables/merge_insert.md).
-
 ## Deleting from a table

 Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter.

 === "Python"

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_row_async"
-        ```
+    ```python
+    tbl.delete('item = "fizz"')
+    ```

    ### Deleting row with specific column value

-    === "Sync API"
+    ```python
+    import lancedb

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row"
-        ```
-    === "Async API"
+    data = [{"x": 1, "vector": [1, 2]},
+            {"x": 2, "vector": [3, 4]},
+            {"x": 3, "vector": [5, 6]}]
+    db = lancedb.connect("./.lancedb")
+    table = db.create_table("my_table", data)
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  2  [3.0, 4.0]
+    # 2  3  [5.0, 6.0]

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_specific_row_async"
-        ```
+    table.delete("x = 2")
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  3  [5.0, 6.0]
+    ```

    ### Delete from a list of values
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values"
-        ```
-    === "Async API"
+    ```python
+    to_remove = [1, 5]
+    to_remove = ", ".join(str(v) for v in to_remove)

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:delete_list_values_async"
-        ```
+    table.delete(f"x IN ({to_remove})")
+    table.to_pandas()
+    #   x      vector
+    # 0  3  [5.0, 6.0]
+    ```

 === "Typescript[^1]"

@@ -724,20 +659,27 @@ This can be used to update zero to all rows depending on how many rows match the
 === "Python"

    API Reference: [lancedb.table.Table.update][]
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table"
-        ```
-    === "Async API"
+    ```python
+    import lancedb
+    import pandas as pd

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-lancedb"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pandas"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_async"
-        ```
+    # Create a lancedb connection
+    db = lancedb.connect("./.lancedb")
+
+    # Create a table from a pandas DataFrame
+    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    table = db.create_table("my_table", data)
+
+    # Update the table where x = 2
+    table.update(where="x = 2", values={"vector": [10, 10]})
+
+    # Get the updated table as a pandas DataFrame
+    df = table.to_pandas()
+
+    # Print the DataFrame
+    print(df)
+    ```

    Output
    ```shell
@@ -792,16 +734,13 @@ This can be used to update zero to all rows depending on how many rows match the
  The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql"
-        ```
-    === "Async API"
+    ```python
+    # Update the table where x = 2
+    table.update(valuesSql={"x": "x + 1"})

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:update_table_sql_async"
-        ```
+    print(table.to_pandas())
+    ```

    Output
    ```shell
@@ -832,16 +771,11 @@ This can be used to update zero to all rows depending on how many rows match the
 Use the `drop_table()` method on the database to remove a table.

 === "Python"
-    === "Sync API"

-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-        ```
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```

      This permanently removes the table and is not recoverable, unlike deleting rows.
      By default, if the table does not exist an exception is raised. To suppress this,
@@ -856,123 +790,6 @@ Use the `drop_table()` method on the database to remove a table.
      This permanently removes the table and is not recoverable, unlike deleting rows.
      If the table does not exist an exception is raised.

-## Changing schemas
-
-While tables must have a schema specified when they are created, you can
-change the schema over time. There's three methods to alter the schema of
-a table:
-
-* `add_columns`: Add new columns to the table
-* `alter_columns`: Alter the name, nullability, or data type of a column
-* `drop_columns`: Drop columns from the table
-
-### Adding new columns
-
-You can add new columns to the table with the `add_columns` method. New columns
-are filled with values based on a SQL expression. For example, you can add a new
-column `y` to the table, fill it with the value of `x * 2` and set the expected
-data type for it.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:add_columns_async"
-        ```
-    **API Reference:** [lancedb.table.Table.add_columns][]
-
-=== "Typescript"
-
-    ```typescript
-    --8<-- "nodejs/examples/basic.test.ts:add_columns"
-    ```
-    **API Reference:** [lancedb.Table.addColumns](../js/classes/Table.md/#addcolumns)
-
-If you want to fill it with null, you can use `cast(NULL as <data_type>)` as
-the SQL expression to fill the column with nulls, while controlling the data
-type of the column. Available data types are base on the
-[DataFusion data types](https://datafusion.apache.org/user-guide/sql/data_types.html).
-You can use any of the SQL types, such as `BIGINT`:
-
-```sql
-cast(NULL as BIGINT)
-```
-
-Using Arrow data types and the `arrow_typeof` function is not yet supported.
-
-<!-- TODO: we could provide a better formula for filling with nulls:
-   https://github.com/lancedb/lance/issues/3175
-->
-
-### Altering existing columns
-
-You can alter the name, nullability, or data type of a column with the `alter_columns`
-method.
-
-Changing the name or nullability of a column just updates the metadata. Because
-of this, it's a fast operation. Changing the data type of a column requires
-rewriting the column, which can be a heavy operation.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_basic.py:alter_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-pyarrow"
-        --8<-- "python/python/tests/docs/test_basic.py:alter_columns_async"
-        ```
-    **API Reference:** [lancedb.table.Table.alter_columns][]
-
-=== "Typescript"
-
-    ```typescript
-    --8<-- "nodejs/examples/basic.test.ts:alter_columns"
-    ```
-    **API Reference:** [lancedb.Table.alterColumns](../js/classes/Table.md/#altercolumns)
-
-### Dropping columns
-
-You can drop columns from the table with the `drop_columns` method. This will
-will remove the column from the schema.
-
-<!-- TODO: Provide guidance on how to reduce disk usage once optimize helps here
-    waiting on: https://github.com/lancedb/lance/issues/3177
-->
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_columns"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_basic.py:drop_columns_async"
-        ```
-    **API Reference:** [lancedb.table.Table.drop_columns][]
-
-=== "Typescript"
-
-    ```typescript
-    --8<-- "nodejs/examples/basic.test.ts:drop_columns"
-    ```
-    **API Reference:** [lancedb.Table.dropColumns](../js/classes/Table.md/#altercolumns)
-
-
 ## Handling bad vectors

 In LanceDB Python, you can use the `on_bad_vectors` parameter to choose how
@@ -1013,46 +830,31 @@ There are three possible settings for `read_consistency_interval`:

    To set strong consistency, use `timedelta(0)`:

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_strong_consistency"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_strong_consistency"
-        ```
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
+    table = db.open_table("my_table")
+    ```

    For eventual consistency, use a custom `timedelta`:

-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_eventual_consistency"
-        ```
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:import-datetime"
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_eventual_consistency"
-        ```
+    ```python
+    from datetime import timedelta
+    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
+    table = db.open_table("my_table")
+    ```

    By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:

-    === "Sync API"
+    ```python
+    db = lancedb.connect("./.lancedb")
+    table = db.open_table("my_table")

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_checkout_latest"
-        ```
-    === "Async API"
+    # (Other writes happen to my_table from another process)

-        ```python
-        --8<-- "python/python/tests/docs/test_guide_tables.py:table_async_checkout_latest"
-        ```
+    # Check for updates
+    table.checkout_latest()
+    ```

 === "Typescript[^1]"

@@ -1060,14 +862,14 @@ There are three possible settings for `read_consistency_interval`:

    ```ts
    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
-    const tbl = await db.openTable("my_table");
+    const table = await db.openTable("my_table");
    ```

    For eventual consistency, specify the update interval as seconds:

    ```ts
    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
-    const tbl = await db.openTable("my_table");
+    const table = await db.openTable("my_table");
    ```

 <!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007
--- a/docs/src/guides/tables/merge_insert.md
+++ b/docs/src/guides/tables/merge_insert.md
@@ -1,135 +0,0 @@
-The merge insert command is a flexible API that can be used to perform:
-
-1. Upsert
-2. Insert-if-not-exists
-3. Replace range
-
-It works by joining the input data with the target table on a key you provide.
-Often this key is a unique row id key. You can then specify what to do when
-there is a match and when there is not a match. For example, for upsert you want
-to update if the row has a match and insert if the row doesn't have a match.
-Whereas for insert-if-not-exists you only want to insert if the row doesn't have
-a match.
-
-You can also read more in the API reference:
-
-* Python
-    * Sync: [lancedb.table.Table.merge_insert][]
-    * Async: [lancedb.table.AsyncTable.merge_insert][]
-* Typescript: [lancedb.Table.mergeInsert](../../js/classes/Table.md/#mergeinsert)
-
-!!! tip "Use scalar indices to speed up merge insert"
-
-    The merge insert command needs to perform a join between the input data and the
-    target table on the `on` key you provide. This requires scanning that entire
-    column, which can be expensive for large tables. To speed up this operation,
-    you can create a scalar index on the `on` column, which will allow LanceDB to
-    find matches without having to scan the whole tables.
-
-    Read more about scalar indices in [Building a Scalar Index](../scalar_index.md)
-    guide.
-
-!!! info "Embedding Functions"
-
-    Like the create table and add APIs, the merge insert API will automatically
-    compute embeddings if the table has a embedding definition in its schema.
-    If the input data doesn't contain the source column, or the vector column
-    is already filled, then the embeddings won't be computed. See the
-    [Embedding Functions](../../embeddings/embedding_functions.md) guide for more
-    information.
-
-## Upsert
-
-Upsert updates rows if they exist and inserts them if they don't. To do this
-with merge insert, enable both `when_matched_update_all()` and
-`when_not_matched_insert_all()`.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:upsert_basic_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:upsert_basic"
-        ```
-
-!!! note "Providing subsets of columns"
-
-    If a column is nullable, it can be omitted from input data and it will be
-    considered `null`. Columns can also be provided in any order.
-
-## Insert-if-not-exists
-
-To avoid inserting duplicate rows, you can use the insert-if-not-exists command.
-This will only insert rows that do not have a match in the target table. To do
-this with merge insert, enable just `when_not_matched_insert_all()`.
-
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:insert_if_not_exists_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:insert_if_not_exists"
-        ```
-
-
-## Replace range
-
-You can also replace a range of rows in the target table with the input data.
-For example, if you have a table of document chunks, where each chunk has
-both a `doc_id` and a `chunk_id`, you can replace all chunks for a given
-`doc_id` with updated chunks. This can be tricky otherwise because if you
-try to use upsert when the new data has fewer chunks you will end up with
-extra chunks. To avoid this, add another clause to delete any chunks for
-the document that are not in the new data, with
-`when_not_matched_by_source_delete`.
-
-=== "Python"
-
-    === "Sync API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range"
-        ```
-
-    === "Async API"
-
-        ```python
-        --8<-- "python/python/tests/docs/test_merge_insert.py:replace_range_async"
-        ```
-
-=== "Typescript"
-
-    === "@lancedb/lancedb"
-
-        ```typescript
-        --8<-- "nodejs/examples/merge_insert.test.ts:replace_range"
-        ```
--- a/docs/src/guides/tuning_retrievers/1_query_types.md
+++ b/docs/src/guides/tuning_retrievers/1_query_types.md
@@ -1,8 +1,8 @@
 ## Improving retriever performance

-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>

-VectorDBs are used as retrievers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retrievers are a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.

 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:

@@ -19,7 +19,7 @@ Using different embedding models is something that's very specific to the use ca


 ## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv).
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)

 ### Using different query types
 Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
@@ -45,14 +45,14 @@ table.add(df[["context"]].to_dict(orient="records"))
 queries = df["query"].tolist()
 ```

-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset:
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.

 * <b> Vector Search: </b>

    ```python
    table.search(quries[0], query_type="vector").limit(5).to_pandas()
    ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement:
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.

    ```python
    table.search(quries[0]).limit(5).to_pandas()
@@ -77,7 +77,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run

 * <b> Hybrid Search: </b>

-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset:
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
    ```python
    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
    ```
@@ -87,7 +87,7 @@ Now that we have the dataset and embeddings table set up, here's how you can run

    !!! note "Note"
        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/).
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)

    

--- a/docs/src/guides/tuning_retrievers/2_reranking.md
+++ b/docs/src/guides/tuning_retrievers/2_reranking.md
@@ -1,6 +1,6 @@
 Continuing from the previous section, we can now rerank the results using more complex rerankers.

-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>

 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:
@@ -62,6 +62,9 @@ Let us take a look at the same datasets from the previous sections, using the sa
 | Reranked fts  | 0.672    |
 | Hybrid | 0.759 |

+### SQuAD Dataset
+
+
 ### Uber10K sec filing Dataset

 | Query Type | Hit-rate@5 |
--- a/docs/src/guides/tuning_retrievers/3_embed_tuning.md
+++ b/docs/src/guides/tuning_retrievers/3_embed_tuning.md
@@ -1,5 +1,5 @@
 ## Finetuning the Embedding Model
-Try it yourself: <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>

 Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.

@@ -16,7 +16,7 @@ validation_df.to_csv("data_val.csv", index=False)
 You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 


-We parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node:
+Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
 ```python
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.readers.file import PagedCSVReader
@@ -43,7 +43,7 @@ val_dataset = generate_qa_embedding_pairs(
 )
 ```

-Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model:
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 

 ```python
 from llama_index.finetuning import SentenceTransformersFinetuneEngine
@@ -57,7 +57,7 @@ finetune_engine = SentenceTransformersFinetuneEngine(
 finetune_engine.finetune()
 embed_model = finetune_engine.get_finetuned_model()
 ```
-This saves the fine tuned embedding model in `tuned_model` folder.
+This saves the fine tuned embedding model in `tuned_model` folder. This al

 # Evaluation results
 In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 
--- a/docs/src/hybrid_search/eval.md
+++ b/docs/src/hybrid_search/eval.md
@@ -3,22 +3,22 @@
 Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.

 ## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step: reranking.
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.

-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example: Weighted linear combination of semantic search & keyword-based search results.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.

-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result-query pair. Example: Cross Encoder models
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models

-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset or application specific so it's hard to generalize.
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.

 ### Example evaluation of hybrid search with Reranking

-Here's some evaluation numbers from an experiment comparing these rerankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.

 <b> With OpenAI ada2 embedding </b>

-Vector Search baseline: `0.64`
+Vector Search baseline - `0.64`

 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
@@ -33,7 +33,7 @@ Vector Search baseline: `0.64`

 <b> With OpenAI embedding-v3-small </b>

-Vector Search baseline: `0.59`
+Vector Search baseline - `0.59`

 | Reranker | Top-3 | Top-5 | Top-10 |
 | --- | --- | --- | --- |
--- a/docs/src/hybrid_search/hybrid_search.md
+++ b/docs/src/hybrid_search/hybrid_search.md
@@ -5,46 +5,57 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .

-=== "Sync API"
+```python
+import os

-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search"
-    ```
-=== "Async API"
+import lancedb
+import openai
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector

-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:import-os"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb"
-    --8<-- "python/python/tests/docs/test_search.py:import-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:import-pydantic"
-    --8<-- "python/python/tests/docs/test_search.py:import-lancedb-fts"
-    --8<-- "python/python/tests/docs/test_search.py:import-openai-embeddings"
-    --8<-- "python/python/tests/docs/test_search.py:class-Documents"
-    --8<-- "python/python/tests/docs/test_search.py:basic_hybrid_search_async"
-    ```
+db = lancedb.connect("~/.lancedb")

+# Ingest embedding function in LanceDB table
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+    openai.api_key = "sk-..."
+embeddings = get_registry().get("openai").create()
+
+class Documents(LanceModel):
+    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+    text: str = embeddings.SourceField()
+
+table = db.create_table("documents", schema=Documents)
+
+data = [
+    { "text": "rebel spaceships striking from a hidden base"},
+    { "text": "have won their first victory against the evil Galactic Empire"},
+    { "text": "during the battle rebel spies managed to steal secret plans"},
+    { "text": "to the Empire's ultimate weapon the Death Star"}
+]
+
+# ingest docs with auto-vectorization
+table.add(data)
+
+# Create a fts index before the hybrid search
+table.create_fts_index("text")
+# hybrid search with default re-ranker
+results = table.search("flower moon", query_type="hybrid").to_pandas()
+```
 !!! Note
    You can also pass the vector and text query manually. This is useful if you're not using the embedding API or if you're using a separate embedder service.
 ### Explicitly passing the vector and text query
-=== "Sync API"
+```python
+vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
+text_query = "flower moon"
+results = table.search(query_type="hybrid")
+                .vector(vector_query)
+                .text(text_query)
+                .limit(5)
+                .to_pandas()

-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text"
-    ```
-=== "Async API"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_search.py:hybrid_search_pass_vector_text_async"
-    ```
+```

 By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion score, to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:

@@ -57,7 +68,7 @@ By default, LanceDB uses `RRFReranker()`, which uses reciprocal rank fusion scor


 ## Available Rerankers
-LanceDB provides a number of rerankers out of the box. You can use any of these rerankers by passing them to the `rerank()` method. 
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. 
 Go to [Rerankers](../reranking/index.md) to learn more about using the available rerankers and implementing custom rerankers.


--- a/docs/src/js/.nojekyll
+++ b/docs/src/js/.nojekyll
@@ -0,0 +1 @@
+TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.
--- a/docs/src/js/README.md
+++ b/docs/src/js/README.md
@@ -36,8 +36,41 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```

-The [quickstart](https://lancedb.github.io/lancedb/basic/) contains a more complete example.
+The [quickstart](../basic.md) contains a more complete example.

 ## Development

-See [CONTRIBUTING.md](_media/CONTRIBUTING.md) for information on how to contribute to LanceDB.
+```sh
+npm run build
+npm run test
+```
+
+### Running lint / format
+
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
+To manually lint your code you can run:
+
+```sh
+npm run lint
+```
+
+to automatically fix all fixable issues:
+
+```sh
+npm run lint-fix
+```
+
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
+
+### Generating docs
+
+```sh
+npm run docs
+
+cd ../docs
+# Asssume the virtual environment was created
+# python3 -m venv venv
+# pip install -r requirements.txt
+. ./venv/bin/activate
+mkdocs build
+```
--- a/docs/src/js/_media/CONTRIBUTING.md
+++ b/docs/src/js/_media/CONTRIBUTING.md
@@ -1,76 +0,0 @@
-# Contributing to LanceDB Typescript
-
-This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
-
-## Project layout
-
-The Typescript package is a wrapper around the Rust library, `lancedb`. We use
-the [napi-rs](https://napi.rs/) library to create the bindings between Rust and
-Typescript.
-
-* `src/`: Rust bindings source code
-* `lancedb/`: Typescript package source code
-* `__test__/`: Unit tests
-* `examples/`: An npm package with the examples shown in the documentation
-
-## Development environment
-
-To set up your development environment, you will need to install the following:
-
-1. Node.js 14 or later
-2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
-3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
-
-Initial setup:
-
-```shell
-npm install
-```
-
-### Commit Hooks
-
-It is **highly recommended** to install the [pre-commit](https://pre-commit.com/) hooks to ensure that your
-code is formatted correctly and passes basic checks before committing:
-
-```shell
-pre-commit install
-```
-
-## Development
-
-Most common development commands can be run using the npm scripts.
-
-Build the package
-
-```shell
-npm install
-npm run build
-```
-
-Lint:
-
-```shell
-npm run lint
-```
-
-Format and fix lints:
-
-```shell
-npm run lint-fix
-```
-
-Run tests:
-
-```shell
-npm test
-```
-
-To run a single test:
-
-```shell
-# Single file: table.test.ts
-npm test -- table.test.ts
-# Single test: 'merge insert' in table.test.ts
-npm test -- table.test.ts --testNamePattern=merge\ insert
-```
--- a/docs/src/js/classes/Connection.md
+++ b/docs/src/js/classes/Connection.md
@@ -23,13 +23,21 @@ be closed when they are garbage collected.
 Any created tables are independent and will continue to work even if
 the underlying connection has been closed.

+## Constructors
+
+### new Connection()
+
+> **new Connection**(): [`Connection`](Connection.md)
+
+#### Returns
+
+[`Connection`](Connection.md)
+
 ## Methods

 ### close()

-```ts
-abstract close(): void
-```
+> `abstract` **close**(): `void`

 Close the connection, releasing any underlying resources.

@@ -45,24 +53,21 @@ Any attempt to use the connection after it is closed will result in an error.

 ### createEmptyTable()

-```ts
-abstract createEmptyTable(
-   name,
-   schema,
-   options?): Promise<Table>
-```
+> `abstract` **createEmptyTable**(`name`, `schema`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;

 Creates a new empty Table

 #### Parameters

-* **name**: `string`
-    The name of the table.
+• **name**: `string`

-* **schema**: [`SchemaLike`](../type-aliases/SchemaLike.md)
-    The schema of the table
+The name of the table.

-* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+• **schema**: `SchemaLike`
+
+The schema of the table
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;

 #### Returns

@@ -74,16 +79,15 @@ Creates a new empty Table

 #### createTable(options)

-```ts
-abstract createTable(options): Promise<Table>
-```
+> `abstract` **createTable**(`options`): `Promise`&lt;[`Table`](Table.md)&gt;

 Creates a new Table and initialize it with new data.

 ##### Parameters

-* **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
-    The options object.
+• **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+
+The options object.

 ##### Returns

@@ -91,25 +95,22 @@ Creates a new Table and initialize it with new data.

 #### createTable(name, data, options)

-```ts
-abstract createTable(
-   name,
-   data,
-   options?): Promise<Table>
-```
+> `abstract` **createTable**(`name`, `data`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;

 Creates a new Table and initialize it with new data.

 ##### Parameters

-* **name**: `string`
-    The name of the table.
+• **name**: `string`

-* **data**: [`TableLike`](../type-aliases/TableLike.md) \| `Record`&lt;`string`, `unknown`&gt;[]
-    Non-empty Array of Records
-    to be inserted into the table
+The name of the table.

-* **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+
+Non-empty Array of Records
+to be inserted into the table
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;

 ##### Returns

@@ -119,9 +120,7 @@ Creates a new Table and initialize it with new data.

 ### display()

-```ts
-abstract display(): string
-```
+> `abstract` **display**(): `string`

 Return a brief description of the connection

@@ -131,32 +130,17 @@ Return a brief description of the connection

 ***

-### dropAllTables()
-
-```ts
-abstract dropAllTables(): Promise<void>
-```
-
-Drop all tables in the database.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
 ### dropTable()

-```ts
-abstract dropTable(name): Promise<void>
-```
+> `abstract` **dropTable**(`name`): `Promise`&lt;`void`&gt;

 Drop an existing table.

 #### Parameters

-* **name**: `string`
-    The name of the table to drop.
+• **name**: `string`
+
+The name of the table to drop.

 #### Returns

@@ -166,9 +150,7 @@ Drop an existing table.

 ### isOpen()

-```ts
-abstract isOpen(): boolean
-```
+> `abstract` **isOpen**(): `boolean`

 Return true if the connection has not been closed

@@ -180,18 +162,17 @@ Return true if the connection has not been closed

 ### openTable()

-```ts
-abstract openTable(name, options?): Promise<Table>
-```
+> `abstract` **openTable**(`name`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;

 Open a table in the database.

 #### Parameters

-* **name**: `string`
-    The name of the table
+• **name**: `string`

-* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
+The name of the table
+
+• **options?**: `Partial`&lt;`OpenTableOptions`&gt;

 #### Returns

@@ -201,9 +182,7 @@ Open a table in the database.

 ### tableNames()

-```ts
-abstract tableNames(options?): Promise<string[]>
-```
+> `abstract` **tableNames**(`options`?): `Promise`&lt;`string`[]&gt;

 List all the table names in this database.

@@ -211,9 +190,10 @@ Tables will be returned in lexicographical order.

 #### Parameters

-* **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
-    options to control the
-    paging / start point
+• **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
+
+options to control the
+paging / start point

 #### Returns

--- a/docs/src/js/classes/Index.md
+++ b/docs/src/js/classes/Index.md
@@ -8,30 +8,9 @@

 ## Methods

-### bitmap()
-
-```ts
-static bitmap(): Index
-```
-
-Create a bitmap index.
-
-A `Bitmap` index stores a bitmap for each distinct value in the column for every row.
-
-This index works best for low-cardinality columns, where the number of unique values
-is small (i.e., less than a few hundreds).
-
-#### Returns
-
-[`Index`](Index.md)
-
-***
-
 ### btree()

-```ts
-static btree(): Index
-```
+> `static` **btree**(): [`Index`](Index.md)

 Create a btree index

@@ -57,80 +36,9 @@ block size may be added in the future.

 ***

-### fts()
-
-```ts
-static fts(options?): Index
-```
-
-Create a full text search index
-
-A full text search index is an index on a string column, so that you can conduct full
-text searches on the column.
-
-The results of a full text search are ordered by relevance measured by BM25.
-
-You can combine filters with full text search.
-
-#### Parameters
-
-* **options?**: `Partial`&lt;[`FtsOptions`](../interfaces/FtsOptions.md)&gt;
-
-#### Returns
-
-[`Index`](Index.md)
-
-***
-
-### hnswPq()
-
-```ts
-static hnswPq(options?): Index
-```
-
-Create a hnswPq index
-
-HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
-It is a variant of the HNSW algorithm that uses product quantization to compress
-the vectors.
-
-#### Parameters
-
-* **options?**: `Partial`&lt;[`HnswPqOptions`](../interfaces/HnswPqOptions.md)&gt;
-
-#### Returns
-
-[`Index`](Index.md)
-
-***
-
-### hnswSq()
-
-```ts
-static hnswSq(options?): Index
-```
-
-Create a hnswSq index
-
-HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
-It is a variant of the HNSW algorithm that uses scalar quantization to compress
-the vectors.
-
-#### Parameters
-
-* **options?**: `Partial`&lt;[`HnswSqOptions`](../interfaces/HnswSqOptions.md)&gt;
-
-#### Returns
-
-[`Index`](Index.md)
-
-***
-
 ### ivfPq()

-```ts
-static ivfPq(options?): Index
-```
+> `static` **ivfPq**(`options`?): [`Index`](Index.md)

 Create an IvfPq index

@@ -155,25 +63,29 @@ currently is also a memory intensive operation.

 #### Parameters

-* **options?**: `Partial`&lt;[`IvfPqOptions`](../interfaces/IvfPqOptions.md)&gt;
+• **options?**: `Partial`&lt;[`IvfPqOptions`](../interfaces/IvfPqOptions.md)&gt;

 #### Returns

 [`Index`](Index.md)

-***
+### fts()

-### labelList()
+> `static` **fts**(`options`?): [`Index`](Index.md)

-```ts
-static labelList(): Index
-```
+Create a full text search index

-Create a label list index.
+This index is used to search for text data.  The index is created by tokenizing the text
+into words and then storing occurrences of these words in a data structure called inverted index
+that allows for fast search.

-LabelList index is a scalar index that can be used on `List<T>` columns to
-support queries with `array_contains_all` and `array_contains_any`
-using an underlying bitmap index.
+During a search the query is tokenized and the inverted index is used to find the rows that
+contain the query words.  The rows are then scored based on BM25 and the top scoring rows are
+sorted and returned.
+
+#### Parameters
+
+• **options?**: `Partial`&lt;[`FtsOptions`](../interfaces/FtsOptions.md)&gt;

 #### Returns

--- a/docs/src/js/classes/MakeArrowTableOptions.md
+++ b/docs/src/js/classes/MakeArrowTableOptions.md
@@ -12,13 +12,11 @@ Options to control the makeArrowTable call.

 ### new MakeArrowTableOptions()

-```ts
-new MakeArrowTableOptions(values?): MakeArrowTableOptions
-```
+> **new MakeArrowTableOptions**(`values`?): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)

 #### Parameters

-* **values?**: `Partial`&lt;[`MakeArrowTableOptions`](MakeArrowTableOptions.md)&gt;
+• **values?**: `Partial`&lt;[`MakeArrowTableOptions`](MakeArrowTableOptions.md)&gt;

 #### Returns

@@ -28,9 +26,7 @@ new MakeArrowTableOptions(values?): MakeArrowTableOptions

 ### dictionaryEncodeStrings

-```ts
-dictionaryEncodeStrings: boolean = false;
-```
+> **dictionaryEncodeStrings**: `boolean` = `false`

 If true then string columns will be encoded with dictionary encoding

@@ -44,30 +40,22 @@ If `schema` is provided then this property is ignored.

 ### embeddingFunction?

-```ts
-optional embeddingFunction: EmbeddingFunctionConfig;
-```
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)

 ***

 ### embeddings?

-```ts
-optional embeddings: EmbeddingFunction<unknown, FunctionOptions>;
-```
+> `optional` **embeddings**: [`EmbeddingFunction`](../namespaces/embedding/classes/EmbeddingFunction.md)&lt;`unknown`, `FunctionOptions`&gt;

 ***

 ### schema?

-```ts
-optional schema: SchemaLike;
-```
+> `optional` **schema**: `SchemaLike`

 ***

 ### vectorColumns

-```ts
-vectorColumns: Record<string, VectorColumnOptions>;
-```
+> **vectorColumns**: `Record`&lt;`string`, [`VectorColumnOptions`](VectorColumnOptions.md)&gt;
--- a/docs/src/js/classes/MergeInsertBuilder.md
+++ b/docs/src/js/classes/MergeInsertBuilder.md
@@ -1,126 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / MergeInsertBuilder
-
-# Class: MergeInsertBuilder
-
-A builder used to create and run a merge insert operation
-
-## Constructors
-
-### new MergeInsertBuilder()
-
-```ts
-new MergeInsertBuilder(native, schema): MergeInsertBuilder
-```
-
-Construct a MergeInsertBuilder. __Internal use only.__
-
-#### Parameters
-
-* **native**: `NativeMergeInsertBuilder`
-
-* **schema**: `Schema`&lt;`any`&gt; \| `Promise`&lt;`Schema`&lt;`any`&gt;&gt;
-
-#### Returns
-
-[`MergeInsertBuilder`](MergeInsertBuilder.md)
-
-## Methods
-
-### execute()
-
-```ts
-execute(data): Promise<void>
-```
-
-Executes the merge insert operation
-
-Nothing is returned but the `Table` is updated
-
-#### Parameters
-
-* **data**: [`Data`](../type-aliases/Data.md)
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
-### whenMatchedUpdateAll()
-
-```ts
-whenMatchedUpdateAll(options?): MergeInsertBuilder
-```
-
-Rows that exist in both the source table (new data) and
-the target table (old data) will be updated, replacing
-the old row with the corresponding matching row.
-
-If there are multiple matches then the behavior is undefined.
-Currently this causes multiple copies of the row to be created
-but that behavior is subject to change.
-
-An optional condition may be specified.  If it is, then only
-matched rows that satisfy the condtion will be updated.  Any
-rows that do not satisfy the condition will be left as they
-are.  Failing to satisfy the condition does not cause a
-"matched row" to become a "not matched" row.
-
-The condition should be an SQL string.  Use the prefix
-target. to refer to rows in the target table (old data)
-and the prefix source. to refer to rows in the source
-table (new data).
-
-For example, "target.last_update < source.last_update"
-
-#### Parameters
-
-* **options?**
-
-* **options.where?**: `string`
-
-#### Returns
-
-[`MergeInsertBuilder`](MergeInsertBuilder.md)
-
-***
-
-### whenNotMatchedBySourceDelete()
-
-```ts
-whenNotMatchedBySourceDelete(options?): MergeInsertBuilder
-```
-
-Rows that exist only in the target table (old data) will be
-deleted.  An optional condition can be provided to limit what
-data is deleted.
-
-#### Parameters
-
-* **options?**
-
-* **options.where?**: `string`
-    An optional condition to limit what data is deleted
-
-#### Returns
-
-[`MergeInsertBuilder`](MergeInsertBuilder.md)
-
-***
-
-### whenNotMatchedInsertAll()
-
-```ts
-whenNotMatchedInsertAll(): MergeInsertBuilder
-```
-
-Rows that exist only in the source table (new data) should
-be inserted into the target table.
-
-#### Returns
-
-[`MergeInsertBuilder`](MergeInsertBuilder.md)
--- a/docs/src/js/classes/Query.md
+++ b/docs/src/js/classes/Query.md
@@ -8,21 +8,33 @@

 A builder for LanceDB queries.

-## See
-
-[Table#query](Table.md#query), [Table#search](Table.md#search)
-
 ## Extends

 - [`QueryBase`](QueryBase.md)&lt;`NativeQuery`&gt;

+## Constructors
+
+### new Query()
+
+> **new Query**(`tbl`): [`Query`](Query.md)
+
+#### Parameters
+
+• **tbl**: `Table`
+
+#### Returns
+
+[`Query`](Query.md)
+
+#### Overrides
+
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
+
 ## Properties

 ### inner

-```ts
-protected inner: Query | Promise<Query>;
-```
+> `protected` **inner**: `Query` \| `Promise`&lt;`Query`&gt;

 #### Inherited from

@@ -30,17 +42,47 @@ protected inner: Query | Promise<Query>;

 ## Methods

+### \[asyncIterator\]()
+
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+
+#### Returns
+
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)
+
+***
+
+### doCall()
+
+> `protected` **doCall**(`fn`): `void`
+
+#### Parameters
+
+• **fn**
+
+#### Returns
+
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)
+
+***
+
 ### execute()

-```ts
-protected execute(options?): RecordBatchIterator
-```
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)

 Execute the query and return the results as an

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -66,16 +108,15 @@ single query)

 ### explainPlan()

-```ts
-explainPlan(verbose): Promise<string>
-```
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;

 Generates an explanation of the query execution plan.

 #### Parameters

-* **verbose**: `boolean` = `false`
-    If true, provides a more detailed explanation. Defaults to false.
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.

 #### Returns

@@ -100,44 +141,21 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();

 ***

-### fastSearch()
-
-```ts
-fastSearch(): this
-```
-
-Skip searching un-indexed data. This can make search faster, but will miss
-any data that is not yet indexed.
-
-Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
-
-***
-
 ### ~~filter()~~

-```ts
-filter(predicate): this
-```
+> **filter**(`predicate`): `this`

 A filter statement to be applied to this query.

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

 `this`

-#### See
+#### Alias

 where

@@ -151,33 +169,9 @@ Use `where` instead

 ***

-### fullTextSearch()
-
-```ts
-fullTextSearch(query, options?): this
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
-
-***
-
 ### limit()

-```ts
-limit(limit): this
-```
+> **limit**(`limit`): `this`

 Set the maximum number of results to return.

@@ -186,7 +180,7 @@ called then every valid row from the table will be returned.

 #### Parameters

-* **limit**: `number`
+• **limit**: `number`

 #### Returns

@@ -198,11 +192,27 @@ called then every valid row from the table will be returned.

 ***

+### nativeExecute()
+
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`RecordBatchIterator`&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
+
+***
+
 ### nearestTo()

-```ts
-nearestTo(vector): VectorQuery
-```
+> **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)

 Find the nearest vectors to the given query vector.

@@ -222,7 +232,7 @@ If there is more than one vector column you must use

 #### Parameters

-* **vector**: [`IntoVector`](../type-aliases/IntoVector.md)
+• **vector**: `IntoVector`

 #### Returns

@@ -254,49 +264,9 @@ a default `limit` of 10 will be used.

 ***

-### nearestToText()
-
-```ts
-nearestToText(query, columns?): Query
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **columns?**: `string`[]
-
-#### Returns
-
-[`Query`](Query.md)
-
-***
-
-### offset()
-
-```ts
-offset(offset): this
-```
-
-#### Parameters
-
-* **offset**: `number`
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
-
-***
-
 ### select()

-```ts
-select(columns): this
-```
+> **select**(`columns`): `this`

 Return only the specified columns.

@@ -320,7 +290,7 @@ input to this method would be:

 #### Parameters

-* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;

 #### Returns

@@ -347,15 +317,13 @@ object insertion order is easy to get wrong and `Map` is more foolproof.

 ### toArray()

-```ts
-toArray(options?): Promise<any[]>
-```
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;

 Collect the results as an array of objects.

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -369,15 +337,13 @@ Collect the results as an array of objects.

 ### toArrow()

-```ts
-toArrow(options?): Promise<Table<any>>
-```
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;

 Collect the results as an Arrow

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -395,9 +361,7 @@ ArrowTable.

 ### where()

-```ts
-where(predicate): this
-```
+> **where**(`predicate`): `this`

 A filter statement to be applied to this query.

@@ -405,7 +369,7 @@ The filter should be supplied as an SQL query string.  For example:

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

@@ -425,25 +389,3 @@ on the filter column(s).
 #### Inherited from

 [`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
-
-***
-
-### withRowId()
-
-```ts
-withRowId(): this
-```
-
-Whether to return the row id in the results.
-
-This column can be used to match results between different queries. For
-example, to match results from a full text search and a vector search in
-order to perform hybrid search.
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)
--- a/docs/src/js/classes/QueryBase.md
+++ b/docs/src/js/classes/QueryBase.md
@@ -8,11 +8,6 @@

 Common methods supported by all query types

-## See
-
- - [Query](Query.md)
- - [VectorQuery](VectorQuery.md)
-
 ## Extended by

 - [`Query`](Query.md)
@@ -26,27 +21,65 @@ Common methods supported by all query types

 - `AsyncIterable`&lt;`RecordBatch`&gt;

+## Constructors
+
+### new QueryBase()
+
+> `protected` **new QueryBase**&lt;`NativeQueryType`&gt;(`inner`): [`QueryBase`](QueryBase.md)&lt;`NativeQueryType`&gt;
+
+#### Parameters
+
+• **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
+
+#### Returns
+
+[`QueryBase`](QueryBase.md)&lt;`NativeQueryType`&gt;
+
 ## Properties

 ### inner

-```ts
-protected inner: NativeQueryType | Promise<NativeQueryType>;
-```
+> `protected` **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;

 ## Methods

+### \[asyncIterator\]()
+
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+
+#### Returns
+
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+
+#### Implementation of
+
+`AsyncIterable.[asyncIterator]`
+
+***
+
+### doCall()
+
+> `protected` **doCall**(`fn`): `void`
+
+#### Parameters
+
+• **fn**
+
+#### Returns
+
+`void`
+
+***
+
 ### execute()

-```ts
-protected execute(options?): RecordBatchIterator
-```
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)

 Execute the query and return the results as an

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -68,16 +101,15 @@ single query)

 ### explainPlan()

-```ts
-explainPlan(verbose): Promise<string>
-```
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;

 Generates an explanation of the query execution plan.

 #### Parameters

-* **verbose**: `boolean` = `false`
-    If true, provides a more detailed explanation. Defaults to false.
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.

 #### Returns

@@ -98,40 +130,21 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();

 ***

-### fastSearch()
-
-```ts
-fastSearch(): this
-```
-
-Skip searching un-indexed data. This can make search faster, but will miss
-any data that is not yet indexed.
-
-Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
-
-#### Returns
-
-`this`
-
-***
-
 ### ~~filter()~~

-```ts
-filter(predicate): this
-```
+> **filter**(`predicate`): `this`

 A filter statement to be applied to this query.

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

 `this`

-#### See
+#### Alias

 where

@@ -141,29 +154,9 @@ Use `where` instead

 ***

-### fullTextSearch()
-
-```ts
-fullTextSearch(query, options?): this
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
-
-#### Returns
-
-`this`
-
-***
-
 ### limit()

-```ts
-limit(limit): this
-```
+> **limit**(`limit`): `this`

 Set the maximum number of results to return.

@@ -172,7 +165,7 @@ called then every valid row from the table will be returned.

 #### Parameters

-* **limit**: `number`
+• **limit**: `number`

 #### Returns

@@ -180,27 +173,23 @@ called then every valid row from the table will be returned.

 ***

-### offset()
+### nativeExecute()

-```ts
-offset(offset): this
-```
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;

 #### Parameters

-* **offset**: `number`
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

-`this`
+`Promise`&lt;`RecordBatchIterator`&gt;

 ***

 ### select()

-```ts
-select(columns): this
-```
+> **select**(`columns`): `this`

 Return only the specified columns.

@@ -224,7 +213,7 @@ input to this method would be:

 #### Parameters

-* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;

 #### Returns

@@ -247,15 +236,13 @@ object insertion order is easy to get wrong and `Map` is more foolproof.

 ### toArray()

-```ts
-toArray(options?): Promise<any[]>
-```
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;

 Collect the results as an array of objects.

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -265,15 +252,13 @@ Collect the results as an array of objects.

 ### toArrow()

-```ts
-toArrow(options?): Promise<Table<any>>
-```
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;

 Collect the results as an Arrow

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -287,9 +272,7 @@ ArrowTable.

 ### where()

-```ts
-where(predicate): this
-```
+> **where**(`predicate`): `this`

 A filter statement to be applied to this query.

@@ -297,7 +280,7 @@ The filter should be supplied as an SQL query string.  For example:

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

@@ -313,21 +296,3 @@ x > 5 OR y = 'test'
 Filtering performance can often be improved by creating a scalar index
 on the filter column(s).
 ```
-
-***
-
-### withRowId()
-
-```ts
-withRowId(): this
-```
-
-Whether to return the row id in the results.
-
-This column can be used to match results between different queries. For
-example, to match results from a full text search and a vector search in
-order to perform hybrid search.
-
-#### Returns
-
-`this`
--- a/docs/src/js/classes/RecordBatchIterator.md
+++ b/docs/src/js/classes/RecordBatchIterator.md
@@ -14,13 +14,11 @@

 ### new RecordBatchIterator()

-```ts
-new RecordBatchIterator(promise?): RecordBatchIterator
-```
+> **new RecordBatchIterator**(`promise`?): [`RecordBatchIterator`](RecordBatchIterator.md)

 #### Parameters

-* **promise?**: `Promise`&lt;`RecordBatchIterator`&gt;
+• **promise?**: `Promise`&lt;`RecordBatchIterator`&gt;

 #### Returns

@@ -30,9 +28,7 @@ new RecordBatchIterator(promise?): RecordBatchIterator

 ### next()

-```ts
-next(): Promise<IteratorResult<RecordBatch<any>, any>>
-```
+> **next**(): `Promise`&lt;`IteratorResult`&lt;`RecordBatch`&lt;`any`&gt;, `any`&gt;&gt;

 #### Returns

--- a/docs/src/js/classes/Table.md
+++ b/docs/src/js/classes/Table.md
@@ -14,20 +14,24 @@ will be freed when the Table is garbage collected.  To eagerly free the cache yo
 can call the `close` method.  Once the Table is closed, it cannot be used for any
 further operations.

-Tables are created using the methods [Connection#createTable](Connection.md#createtable)
-and [Connection#createEmptyTable](Connection.md#createemptytable). Existing tables are opened
-using [Connection#openTable](Connection.md#opentable).
-
 Closing a table is optional.  It not closed, it will be closed when it is garbage
 collected.

+## Constructors
+
+### new Table()
+
+> **new Table**(): [`Table`](Table.md)
+
+#### Returns
+
+[`Table`](Table.md)
+
 ## Accessors

 ### name

-```ts
-get abstract name(): string
-```
+> `get` `abstract` **name**(): `string`

 Returns the name of the table

@@ -39,18 +43,17 @@ Returns the name of the table

 ### add()

-```ts
-abstract add(data, options?): Promise<void>
-```
+> `abstract` **add**(`data`, `options`?): `Promise`&lt;`void`&gt;

 Insert records into this Table.

 #### Parameters

-* **data**: [`Data`](../type-aliases/Data.md)
-    Records to be inserted into the Table
+• **data**: [`Data`](../type-aliases/Data.md)

-* **options?**: `Partial`&lt;[`AddDataOptions`](../interfaces/AddDataOptions.md)&gt;
+Records to be inserted into the Table
+
+• **options?**: `Partial`&lt;[`AddDataOptions`](../interfaces/AddDataOptions.md)&gt;

 #### Returns

@@ -60,19 +63,18 @@ Insert records into this Table.

 ### addColumns()

-```ts
-abstract addColumns(newColumnTransforms): Promise<void>
-```
+> `abstract` **addColumns**(`newColumnTransforms`): `Promise`&lt;`void`&gt;

 Add new columns with defined values.

 #### Parameters

-* **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
-    pairs of column names and
-    the SQL expression to use to calculate the value of the new column. These
-    expressions will be evaluated for each row in the table, and can
-    reference existing columns in the table.
+• **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
+
+pairs of column names and
+the SQL expression to use to calculate the value of the new column. These
+expressions will be evaluated for each row in the table, and can
+reference existing columns in the table.

 #### Returns

@@ -82,17 +84,16 @@ Add new columns with defined values.

 ### alterColumns()

-```ts
-abstract alterColumns(columnAlterations): Promise<void>
-```
+> `abstract` **alterColumns**(`columnAlterations`): `Promise`&lt;`void`&gt;

 Alter the name or nullability of columns.

 #### Parameters

-* **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
-    One or more alterations to
-    apply to columns.
+• **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
+
+One or more alterations to
+apply to columns.

 #### Returns

@@ -102,9 +103,7 @@ Alter the name or nullability of columns.

 ### checkout()

-```ts
-abstract checkout(version): Promise<void>
-```
+> `abstract` **checkout**(`version`): `Promise`&lt;`void`&gt;

 Checks out a specific version of the table _This is an in-place operation._

@@ -117,8 +116,9 @@ wish to return to standard mode, call `checkoutLatest`.

 #### Parameters

-* **version**: `number`
-    The version to checkout
+• **version**: `number`
+
+The version to checkout

 #### Returns

@@ -144,9 +144,7 @@ console.log(await table.version()); // 2

 ### checkoutLatest()

-```ts
-abstract checkoutLatest(): Promise<void>
-```
+> `abstract` **checkoutLatest**(): `Promise`&lt;`void`&gt;

 Checkout the latest version of the table. _This is an in-place operation._

@@ -161,9 +159,7 @@ version of the table.

 ### close()

-```ts
-abstract close(): void
-```
+> `abstract` **close**(): `void`

 Close the table, releasing any underlying resources.

@@ -179,15 +175,13 @@ Any attempt to use the table after it is closed will result in an error.

 ### countRows()

-```ts
-abstract countRows(filter?): Promise<number>
-```
+> `abstract` **countRows**(`filter`?): `Promise`&lt;`number`&gt;

 Count the total number of rows in the dataset.

 #### Parameters

-* **filter?**: `string`
+• **filter?**: `string`

 #### Returns

@@ -197,9 +191,7 @@ Count the total number of rows in the dataset.

 ### createIndex()

-```ts
-abstract createIndex(column, options?): Promise<void>
-```
+> `abstract` **createIndex**(`column`, `options`?): `Promise`&lt;`void`&gt;

 Create an index to speed up queries.

@@ -208,19 +200,21 @@ Indices on vector columns will speed up vector searches.
 Indices on scalar columns will speed up filtering (in both
 vector and non-vector searches)

-We currently don't support custom named indexes.
-The index name will always be `${column}_idx`.
-
 #### Parameters

-* **column**: `string`
+• **column**: `string`

-* **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;
+• **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;

 #### Returns

 `Promise`&lt;`void`&gt;

+#### Note
+
+We currently don't support custom named indexes,
+The index name will always be `${column}_idx`
+
 #### Examples

 ```ts
@@ -251,15 +245,13 @@ await table.createIndex("my_float_col");

 ### delete()

-```ts
-abstract delete(predicate): Promise<void>
-```
+> `abstract` **delete**(`predicate`): `Promise`&lt;`void`&gt;

 Delete the rows that satisfy the predicate.

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

@@ -269,9 +261,7 @@ Delete the rows that satisfy the predicate.

 ### display()

-```ts
-abstract display(): string
-```
+> `abstract` **display**(): `string`

 Return a brief description of the table

@@ -283,9 +273,7 @@ Return a brief description of the table

 ### dropColumns()

-```ts
-abstract dropColumns(columnNames): Promise<void>
-```
+> `abstract` **dropColumns**(`columnNames`): `Promise`&lt;`void`&gt;

 Drop one or more columns from the dataset

@@ -296,32 +284,11 @@ then call ``cleanup_files`` to remove the old files.

 #### Parameters

-* **columnNames**: `string`[]
-    The names of the columns to drop. These can
-    be nested column references (e.g. "a.b.c") or top-level column names
-    (e.g. "a").
+• **columnNames**: `string`[]

-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
-### dropIndex()
-
-```ts
-abstract dropIndex(name): Promise<void>
-```
-
-Drop an index from the table.
-
-#### Parameters
-
-* **name**: `string`
-    The name of the index.
-    This does not delete the index from disk, it just removes it from the table.
-    To delete the index, run [Table#optimize](Table.md#optimize) after dropping the index.
-    Use [Table.listIndices](Table.md#listindices) to find the names of the indices.
+The names of the columns to drop. These can
+be nested column references (e.g. "a.b.c") or top-level column names
+(e.g. "a").

 #### Returns

@@ -331,16 +298,15 @@ Drop an index from the table.

 ### indexStats()

-```ts
-abstract indexStats(name): Promise<undefined | IndexStatistics>
-```
+> `abstract` **indexStats**(`name`): `Promise`&lt;`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)&gt;

 List all the stats of a specified index

 #### Parameters

-* **name**: `string`
-    The name of the index.
+• **name**: `string`
+
+The name of the index.

 #### Returns

@@ -348,15 +314,11 @@ List all the stats of a specified index

 The stats of the index. If the index does not exist, it will return undefined

-Use [Table.listIndices](Table.md#listindices) to find the names of the indices.
-
 ***

 ### isOpen()

-```ts
-abstract isOpen(): boolean
-```
+> `abstract` **isOpen**(): `boolean`

 Return true if the table has not been closed

@@ -368,9 +330,7 @@ Return true if the table has not been closed

 ### listIndices()

-```ts
-abstract listIndices(): Promise<IndexConfig[]>
-```
+> `abstract` **listIndices**(): `Promise`&lt;[`IndexConfig`](../interfaces/IndexConfig.md)[]&gt;

 List all indices that have been created with [Table.createIndex](Table.md#createindex)

@@ -380,41 +340,23 @@ List all indices that have been created with [Table.createIndex](Table.md#create

 ***

-### listVersions()
-
-```ts
-abstract listVersions(): Promise<Version[]>
-```
-
-List all the versions of the table
-
-#### Returns
-
-`Promise`&lt;[`Version`](../interfaces/Version.md)[]&gt;
-
-***
-
 ### mergeInsert()

-```ts
-abstract mergeInsert(on): MergeInsertBuilder
-```
+> `abstract` **mergeInsert**(`on`): `MergeInsertBuilder`

 #### Parameters

-* **on**: `string` \| `string`[]
+• **on**: `string` \| `string`[]

 #### Returns

-[`MergeInsertBuilder`](MergeInsertBuilder.md)
+`MergeInsertBuilder`

 ***

 ### optimize()

-```ts
-abstract optimize(options?): Promise<OptimizeStats>
-```
+> `abstract` **optimize**(`options`?): `Promise`&lt;`OptimizeStats`&gt;

 Optimize the on-disk data and indices for better performance.

@@ -446,19 +388,17 @@ Modeled after ``VACUUM`` in PostgreSQL.

 #### Parameters

-* **options?**: `Partial`&lt;[`OptimizeOptions`](../interfaces/OptimizeOptions.md)&gt;
+• **options?**: `Partial`&lt;`OptimizeOptions`&gt;

 #### Returns

-`Promise`&lt;[`OptimizeStats`](../interfaces/OptimizeStats.md)&gt;
+`Promise`&lt;`OptimizeStats`&gt;

 ***

 ### query()

-```ts
-abstract query(): Query
-```
+> `abstract` **query**(): [`Query`](Query.md)

 Create a [Query](Query.md) Builder.

@@ -526,9 +466,7 @@ for await (const batch of table.query()) {

 ### restore()

-```ts
-abstract restore(): Promise<void>
-```
+> `abstract` **restore**(): `Promise`&lt;`void`&gt;

 Restore the table to the currently checked out version

@@ -549,9 +487,7 @@ out state and the read_consistency_interval, if any, will apply.

 ### schema()

-```ts
-abstract schema(): Promise<Schema<any>>
-```
+> `abstract` **schema**(): `Promise`&lt;`Schema`&lt;`any`&gt;&gt;

 Get the schema of the table.

@@ -563,41 +499,61 @@ Get the schema of the table.

 ### search()

-```ts
-abstract search(
-   query,
-   queryType?,
-   ftsColumns?): Query | VectorQuery
-```
+#### search(query)
+
+> `abstract` **search**(`query`, `queryType`, `ftsColumns`): [`VectorQuery`](VectorQuery.md)

 Create a search query to find the nearest neighbors
-of the given query
+of the given query vector, or the documents
+with the highest relevance to the query string.

-#### Parameters
+##### Parameters

-* **query**: `string` \| [`IntoVector`](../type-aliases/IntoVector.md)
-    the query, a vector or string
+• **query**: `string`

-* **queryType?**: `string`
-    the type of the query, "vector", "fts", or "auto"
+the query. This will be converted to a vector using the table's provided embedding function,
+or the query string for full-text search if `queryType` is "fts".

-* **ftsColumns?**: `string` \| `string`[]
-    the columns to search in for full text search
-    for now, only one column can be searched at a time.
-    when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
-    if the query is a string and no embedding function is defined, it will be treated as a full text search query
+• **queryType**: `string` =  `"auto"` \| `"fts"`

-#### Returns
+the type of query to run. If "auto", the query type will be determined based on the query.

-[`Query`](Query.md) \| [`VectorQuery`](VectorQuery.md)
+• **ftsColumns**: `string[] | str` =  undefined
+
+the columns to search in. If not provided, all indexed columns will be searched.
+
+For now, this can support to search only one column.
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+##### Note
+
+If no embedding functions are defined in the table, this will error when collecting the results.
+
+#### search(query)
+
+> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
+
+Create a search query to find the nearest neighbors
+of the given query vector
+
+##### Parameters
+
+• **query**: `IntoVector`
+
+the query vector
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)

 ***

 ### toArrow()

-```ts
-abstract toArrow(): Promise<Table<any>>
-```
+> `abstract` **toArrow**(): `Promise`&lt;`Table`&lt;`any`&gt;&gt;

 Return the table as an arrow table

@@ -611,15 +567,13 @@ Return the table as an arrow table

 #### update(opts)

-```ts
-abstract update(opts): Promise<void>
-```
+> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;

 Update existing records in the Table

 ##### Parameters

-* **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;

 ##### Returns

@@ -633,15 +587,13 @@ table.update({where:"x = 2", values:{"vector": [10, 10]}})

 #### update(opts)

-```ts
-abstract update(opts): Promise<void>
-```
+> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;

 Update existing records in the Table

 ##### Parameters

-* **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;

 ##### Returns

@@ -655,9 +607,7 @@ table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})

 #### update(updates, options)

-```ts
-abstract update(updates, options?): Promise<void>
-```
+> `abstract` **update**(`updates`, `options`?): `Promise`&lt;`void`&gt;

 Update existing records in the Table

@@ -676,17 +626,20 @@ repeatedly calilng this method.

 ##### Parameters

-* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
-    the
-    columns to update
-    Keys in the map should specify the name of the column to update.
-    Values in the map provide the new value of the column.  These can
-    be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
-    based on the row being updated (e.g. "my_col + 1")
+• **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;

-* **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
-    additional options to control
-    the update behavior
+the
+columns to update
+
+Keys in the map should specify the name of the column to update.
+Values in the map provide the new value of the column.  These can
+be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+based on the row being updated (e.g. "my_col + 1")
+
+• **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+
+additional options to control
+the update behavior

 ##### Returns

@@ -696,9 +649,7 @@ repeatedly calilng this method.

 ### vectorSearch()

-```ts
-abstract vectorSearch(vector): VectorQuery
-```
+> `abstract` **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)

 Search the table with a given query vector.

@@ -708,7 +659,7 @@ by `query`.

 #### Parameters

-* **vector**: [`IntoVector`](../type-aliases/IntoVector.md)
+• **vector**: `IntoVector`

 #### Returns

@@ -722,12 +673,36 @@ by `query`.

 ### version()

-```ts
-abstract version(): Promise<number>
-```
+> `abstract` **version**(): `Promise`&lt;`number`&gt;

 Retrieve the version of the table

 #### Returns

 `Promise`&lt;`number`&gt;
+
+***
+
+### parseTableData()
+
+> `static` **parseTableData**(`data`, `options`?, `streaming`?): `Promise`&lt;`object`&gt;
+
+#### Parameters
+
+• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+
+• **streaming?**: `boolean` = `false`
+
+#### Returns
+
+`Promise`&lt;`object`&gt;
+
+##### buf
+
+> **buf**: `Buffer`
+
+##### mode
+
+> **mode**: `string`
--- a/docs/src/js/classes/VectorColumnOptions.md
+++ b/docs/src/js/classes/VectorColumnOptions.md
@@ -10,13 +10,11 @@

 ### new VectorColumnOptions()

-```ts
-new VectorColumnOptions(values?): VectorColumnOptions
-```
+> **new VectorColumnOptions**(`values`?): [`VectorColumnOptions`](VectorColumnOptions.md)

 #### Parameters

-* **values?**: `Partial`&lt;[`VectorColumnOptions`](VectorColumnOptions.md)&gt;
+• **values?**: `Partial`&lt;[`VectorColumnOptions`](VectorColumnOptions.md)&gt;

 #### Returns

@@ -26,8 +24,6 @@ new VectorColumnOptions(values?): VectorColumnOptions

 ### type

-```ts
-type: Float<Floats>;
-```
+> **type**: `Float`&lt;`Floats`&gt;

 Vector column type.
--- a/docs/src/js/classes/VectorQuery.md
+++ b/docs/src/js/classes/VectorQuery.md
@@ -10,21 +10,33 @@ A builder used to construct a vector search

 This builder can be reused to execute the query many times.

-## See
-
-[Query#nearestTo](Query.md#nearestto)
-
 ## Extends

 - [`QueryBase`](QueryBase.md)&lt;`NativeVectorQuery`&gt;

+## Constructors
+
+### new VectorQuery()
+
+> **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
+
+#### Parameters
+
+• **inner**: `VectorQuery` \| `Promise`&lt;`VectorQuery`&gt;
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### Overrides
+
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
+
 ## Properties

 ### inner

-```ts
-protected inner: VectorQuery | Promise<VectorQuery>;
-```
+> `protected` **inner**: `VectorQuery` \| `Promise`&lt;`VectorQuery`&gt;

 #### Inherited from

@@ -32,27 +44,23 @@ protected inner: VectorQuery | Promise<VectorQuery>;

 ## Methods

-### addQueryVector()
+### \[asyncIterator\]()

-```ts
-addQueryVector(vector): VectorQuery
-```
-
-#### Parameters
-
-* **vector**: [`IntoVector`](../type-aliases/IntoVector.md)
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;

 #### Returns

-[`VectorQuery`](VectorQuery.md)
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)

 ***

 ### bypassVectorIndex()

-```ts
-bypassVectorIndex(): VectorQuery
-```
+> **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)

 If this is called then any vector index is skipped

@@ -70,9 +78,7 @@ calculate your recall to select an appropriate value for nprobes.

 ### column()

-```ts
-column(column): VectorQuery
-```
+> **column**(`column`): [`VectorQuery`](VectorQuery.md)

 Set the vector column to query

@@ -81,7 +87,7 @@ the call to

 #### Parameters

-* **column**: `string`
+• **column**: `string`

 #### Returns

@@ -96,29 +102,9 @@ whose data type is a fixed-size-list of floats.

 ***

-### distanceRange()
-
-```ts
-distanceRange(lowerBound?, upperBound?): VectorQuery
-```
-
-#### Parameters
-
-* **lowerBound?**: `number`
-
-* **upperBound?**: `number`
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-***
-
 ### distanceType()

-```ts
-distanceType(distanceType): VectorQuery
-```
+> **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)

 Set the distance metric to use

@@ -128,7 +114,7 @@ use.  See

 #### Parameters

-* **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
+• **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`

 #### Returns

@@ -147,41 +133,33 @@ By default "l2" is used.

 ***

-### ef()
+### doCall()

-```ts
-ef(ef): VectorQuery
-```
-
-Set the number of candidates to consider during the search
-
-This argument is only used when the vector column has an HNSW index.
-If there is no index then this value is ignored.
-
-Increasing this value will increase the recall of your query but will
-also increase the latency of your query. The default value is 1.5*limit.
+> `protected` **doCall**(`fn`): `void`

 #### Parameters

-* **ef**: `number`
+• **fn**

 #### Returns

-[`VectorQuery`](VectorQuery.md)
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)

 ***

 ### execute()

-```ts
-protected execute(options?): RecordBatchIterator
-```
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)

 Execute the query and return the results as an

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -207,16 +185,15 @@ single query)

 ### explainPlan()

-```ts
-explainPlan(verbose): Promise<string>
-```
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;

 Generates an explanation of the query execution plan.

 #### Parameters

-* **verbose**: `boolean` = `false`
-    If true, provides a more detailed explanation. Defaults to false.
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.

 #### Returns

@@ -241,44 +218,21 @@ const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();

 ***

-### fastSearch()
-
-```ts
-fastSearch(): this
-```
-
-Skip searching un-indexed data. This can make search faster, but will miss
-any data that is not yet indexed.
-
-Use [Table#optimize](Table.md#optimize) to index all un-indexed data.
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`fastSearch`](QueryBase.md#fastsearch)
-
-***
-
 ### ~~filter()~~

-```ts
-filter(predicate): this
-```
+> **filter**(`predicate`): `this`

 A filter statement to be applied to this query.

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

 `this`

-#### See
+#### Alias

 where

@@ -292,33 +246,9 @@ Use `where` instead

 ***

-### fullTextSearch()
-
-```ts
-fullTextSearch(query, options?): this
-```
-
-#### Parameters
-
-* **query**: `string`
-
-* **options?**: `Partial`&lt;[`FullTextSearchOptions`](../interfaces/FullTextSearchOptions.md)&gt;
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`fullTextSearch`](QueryBase.md#fulltextsearch)
-
-***
-
 ### limit()

-```ts
-limit(limit): this
-```
+> **limit**(`limit`): `this`

 Set the maximum number of results to return.

@@ -327,7 +257,7 @@ called then every valid row from the table will be returned.

 #### Parameters

-* **limit**: `number`
+• **limit**: `number`

 #### Returns

@@ -339,11 +269,27 @@ called then every valid row from the table will be returned.

 ***

+### nativeExecute()
+
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`RecordBatchIterator`&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
+
+***
+
 ### nprobes()

-```ts
-nprobes(nprobes): VectorQuery
-```
+> **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)

 Set the number of partitions to search (probe)

@@ -368,7 +314,7 @@ you the desired recall.

 #### Parameters

-* **nprobes**: `number`
+• **nprobes**: `number`

 #### Returns

@@ -376,31 +322,9 @@ you the desired recall.

 ***

-### offset()
-
-```ts
-offset(offset): this
-```
-
-#### Parameters
-
-* **offset**: `number`
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`offset`](QueryBase.md#offset)
-
-***
-
 ### postfilter()

-```ts
-postfilter(): VectorQuery
-```
+> **postfilter**(): [`VectorQuery`](VectorQuery.md)

 If this is called then filtering will happen after the vector search instead of
 before.
@@ -432,9 +356,7 @@ factor can often help restore some of the results lost by post filtering.

 ### refineFactor()

-```ts
-refineFactor(refineFactor): VectorQuery
-```
+> **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)

 A multiplier to control how many additional rows are taken during the refine step

@@ -466,23 +388,7 @@ distance between the query vector and the actual uncompressed vector.

 #### Parameters

-* **refineFactor**: `number`
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-***
-
-### rerank()
-
-```ts
-rerank(reranker): VectorQuery
-```
-
-#### Parameters
-
-* **reranker**: [`Reranker`](../namespaces/rerankers/interfaces/Reranker.md)
+• **refineFactor**: `number`

 #### Returns

@@ -492,9 +398,7 @@ rerank(reranker): VectorQuery

 ### select()

-```ts
-select(columns): this
-```
+> **select**(`columns`): `this`

 Return only the specified columns.

@@ -518,7 +422,7 @@ input to this method would be:

 #### Parameters

-* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;

 #### Returns

@@ -545,15 +449,13 @@ object insertion order is easy to get wrong and `Map` is more foolproof.

 ### toArray()

-```ts
-toArray(options?): Promise<any[]>
-```
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;

 Collect the results as an array of objects.

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -567,15 +469,13 @@ Collect the results as an array of objects.

 ### toArrow()

-```ts
-toArrow(options?): Promise<Table<any>>
-```
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;

 Collect the results as an Arrow

 #### Parameters

-* **options?**: `Partial`&lt;[`QueryExecutionOptions`](../interfaces/QueryExecutionOptions.md)&gt;
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;

 #### Returns

@@ -593,9 +493,7 @@ ArrowTable.

 ### where()

-```ts
-where(predicate): this
-```
+> **where**(`predicate`): `this`

 A filter statement to be applied to this query.

@@ -603,7 +501,7 @@ The filter should be supplied as an SQL query string.  For example:

 #### Parameters

-* **predicate**: `string`
+• **predicate**: `string`

 #### Returns

@@ -623,25 +521,3 @@ on the filter column(s).
 #### Inherited from

 [`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
-
-***
-
-### withRowId()
-
-```ts
-withRowId(): this
-```
-
-Whether to return the row id in the results.
-
-This column can be used to match results between different queries. For
-example, to match results from a full text search and a vector search in
-order to perform hybrid search.
-
-#### Returns
-
-`this`
-
-#### Inherited from
-
-[`QueryBase`](QueryBase.md).[`withRowId`](QueryBase.md#withrowid)
--- a/docs/src/js/enumerations/WriteMode.md
+++ b/docs/src/js/enumerations/WriteMode.md
@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteMode
+
+# Enumeration: WriteMode
+
+Write mode for writing a table.
+
+## Enumeration Members
+
+### Append
+
+> **Append**: `"Append"`
+
+***
+
+### Create
+
+> **Create**: `"Create"`
+
+***
+
+### Overwrite
+
+> **Overwrite**: `"Overwrite"`
--- a/docs/src/js/functions/connect.md
+++ b/docs/src/js/functions/connect.md
@@ -6,11 +6,9 @@

 # Function: connect()

-## connect(uri, options)
+## connect(uri, opts)

-```ts
-function connect(uri, options?): Promise<Connection>
-```
+> **connect**(`uri`, `opts`?): `Promise`&lt;[`Connection`](../classes/Connection.md)&gt;

 Connect to a LanceDB instance at the given URI.

@@ -22,12 +20,12 @@ Accepted formats:

 ### Parameters

-* **uri**: `string`
-    The uri of the database. If the database uri starts
-    with `db://` then it connects to a remote database.
+• **uri**: `string`

-* **options?**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md)&gt;
-    The options to use when connecting to the database
+The uri of the database. If the database uri starts
+with `db://` then it connects to a remote database.
+
+• **opts?**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`&gt;

 ### Returns

@@ -50,11 +48,9 @@ const conn = await connect(
 });
 ```

-## connect(options)
+## connect(opts)

-```ts
-function connect(options): Promise<Connection>
-```
+> **connect**(`opts`): `Promise`&lt;[`Connection`](../classes/Connection.md)&gt;

 Connect to a LanceDB instance at the given URI.

@@ -66,8 +62,7 @@ Accepted formats:

 ### Parameters

-* **options**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md)&gt; & `object`
-    The options to use when connecting to the database
+• **opts**: `Partial`&lt;[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`&gt; & `object`

 ### Returns

--- a/docs/src/js/functions/makeArrowTable.md
+++ b/docs/src/js/functions/makeArrowTable.md
@@ -6,12 +6,7 @@

 # Function: makeArrowTable()

-```ts
-function makeArrowTable(
-   data,
-   options?,
-   metadata?): ArrowTable
-```
+> **makeArrowTable**(`data`, `options`?, `metadata`?): `ArrowTable`

 An enhanced version of the makeTable function from Apache Arrow
 that supports nested fields and embeddings columns.
@@ -22,6 +17,8 @@ when creating a table or adding data to it)
 This function converts an array of Record<String, any> (row-major JS objects)
 to an Arrow Table (a columnar structure)

+Note that it currently does not support nulls.
+
 If a schema is provided then it will be used to determine the resulting array
 types.  Fields will also be reordered to fit the order defined by the schema.

@@ -29,9 +26,6 @@ If a schema is not provided then the types will be inferred and the field order
 will be controlled by the order of properties in the first record.  If a type
 is inferred it will always be nullable.

-If not all fields are found in the data, then a subset of the schema will be
-returned.
-
 If the input is empty then a schema must be provided to create an empty table.

 When a schema is not specified then data types will be inferred.  The inference
@@ -39,7 +33,6 @@ rules are as follows:

 - boolean => Bool
 - number => Float64
- - bigint => Int64
 - String => Utf8
 - Buffer => Binary
 - Record<String, any> => Struct
@@ -47,11 +40,11 @@ rules are as follows:

 ## Parameters

-* **data**: `Record`&lt;`string`, `unknown`&gt;[]
+• **data**: `Record`&lt;`string`, `unknown`&gt;[]

-* **options?**: `Partial`&lt;[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)&gt;
+• **options?**: `Partial`&lt;[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)&gt;

-* **metadata?**: `Map`&lt;`string`, `string`&gt;
+• **metadata?**: `Map`&lt;`string`, `string`&gt;

 ## Returns

@@ -59,7 +52,6 @@ rules are as follows:

 ## Example

-```ts
 import { fromTableToBuffer, makeArrowTable } from "../arrow";
 import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";

@@ -81,40 +73,42 @@ The `vectorColumns` option can be used to support other vector column
 names and data types.

 ```ts
+
 const schema = new Schema([
-  new Field("a", new Float64()),
-  new Field("b", new Float64()),
-  new Field(
-    "vector",
-    new FixedSizeList(3, new Field("item", new Float32()))
-  ),
-]);
-const table = makeArrowTable([
-  { a: 1, b: 2, vector: [1, 2, 3] },
-  { a: 4, b: 5, vector: [4, 5, 6] },
-  { a: 7, b: 8, vector: [7, 8, 9] },
-]);
-assert.deepEqual(table.schema, schema);
+   new Field("a", new Float64()),
+   new Field("b", new Float64()),
+   new Field(
+     "vector",
+     new FixedSizeList(3, new Field("item", new Float32()))
+   ),
+ ]);
+ const table = makeArrowTable([
+   { a: 1, b: 2, vector: [1, 2, 3] },
+   { a: 4, b: 5, vector: [4, 5, 6] },
+   { a: 7, b: 8, vector: [7, 8, 9] },
+ ]);
+ assert.deepEqual(table.schema, schema);
 ```

 You can specify the vector column types and names using the options as well

-```ts
+```typescript
+
 const schema = new Schema([
-  new Field('a', new Float64()),
-  new Field('b', new Float64()),
-  new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-  new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
-]);
+   new Field('a', new Float64()),
+   new Field('b', new Float64()),
+   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ ]);
 const table = makeArrowTable([
-  { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-  { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-  { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
-], {
-  vectorColumns: {
-    vec1: { type: new Float16() },
-    vec2: { type: new Float16() }
-  }
-}
+   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ ], {
+   vectorColumns: {
+     vec1: { type: new Float16() },
+     vec2: { type: new Float16() }
+   }
+ }
 assert.deepEqual(table.schema, schema)
 ```
--- a/docs/src/js/globals.md
+++ b/docs/src/js/globals.md
@@ -7,14 +7,16 @@
 ## Namespaces

 - [embedding](namespaces/embedding/README.md)
- [rerankers](namespaces/rerankers/README.md)
+
+## Enumerations
+
+- [WriteMode](enumerations/WriteMode.md)

 ## Classes

 - [Connection](classes/Connection.md)
 - [Index](classes/Index.md)
 - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
 - [Query](classes/Query.md)
 - [QueryBase](classes/QueryBase.md)
 - [RecordBatchIterator](classes/RecordBatchIterator.md)
@@ -26,41 +28,23 @@

 - [AddColumnsSql](interfaces/AddColumnsSql.md)
 - [AddDataOptions](interfaces/AddDataOptions.md)
- [ClientConfig](interfaces/ClientConfig.md)
 - [ColumnAlteration](interfaces/ColumnAlteration.md)
- [CompactionStats](interfaces/CompactionStats.md)
 - [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [CreateTableOptions](interfaces/CreateTableOptions.md)
 - [ExecutableQuery](interfaces/ExecutableQuery.md)
- [FtsOptions](interfaces/FtsOptions.md)
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
 - [IndexConfig](interfaces/IndexConfig.md)
+- [IndexMetadata](interfaces/IndexMetadata.md)
 - [IndexOptions](interfaces/IndexOptions.md)
 - [IndexStatistics](interfaces/IndexStatistics.md)
 - [IvfPqOptions](interfaces/IvfPqOptions.md)
- [OpenTableOptions](interfaces/OpenTableOptions.md)
- [OptimizeOptions](interfaces/OptimizeOptions.md)
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RemovalStats](interfaces/RemovalStats.md)
- [RetryConfig](interfaces/RetryConfig.md)
+- [FtsOptions](interfaces/FtsOptions.md)
 - [TableNamesOptions](interfaces/TableNamesOptions.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
 - [UpdateOptions](interfaces/UpdateOptions.md)
- [Version](interfaces/Version.md)
+- [WriteOptions](interfaces/WriteOptions.md)

 ## Type Aliases

 - [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
- [SchemaLike](type-aliases/SchemaLike.md)
- [TableLike](type-aliases/TableLike.md)

 ## Functions

--- a/docs/src/js/interfaces/AddColumnsSql.md
+++ b/docs/src/js/interfaces/AddColumnsSql.md
@@ -12,9 +12,7 @@ A definition of a new column to add to a table.

 ### name

-```ts
-name: string;
-```
+> **name**: `string`

 The name of the new column.

@@ -22,9 +20,7 @@ The name of the new column.

 ### valueSql

-```ts
-valueSql: string;
-```
+> **valueSql**: `string`

 The values to populate the new column with, as a SQL expression.
 The expression can reference other columns in the table.
--- a/docs/src/js/interfaces/AddDataOptions.md
+++ b/docs/src/js/interfaces/AddDataOptions.md
@@ -12,9 +12,7 @@ Options for adding data to a table.

 ### mode

-```ts
-mode: "append" | "overwrite";
-```
+> **mode**: `"append"` \| `"overwrite"`

 If "append" (the default) then the new data will be added to the table

--- a/docs/src/js/interfaces/ClientConfig.md
+++ b/docs/src/js/interfaces/ClientConfig.md
@@ -1,39 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / ClientConfig
-
-# Interface: ClientConfig
-
-## Properties
-
-### extraHeaders?
-
-```ts
-optional extraHeaders: Record<string, string>;
-```
-
-***
-
-### retryConfig?
-
-```ts
-optional retryConfig: RetryConfig;
-```
-
-***
-
-### timeoutConfig?
-
-```ts
-optional timeoutConfig: TimeoutConfig;
-```
-
-***
-
-### userAgent?
-
-```ts
-optional userAgent: string;
-```
--- a/docs/src/js/interfaces/ColumnAlteration.md
+++ b/docs/src/js/interfaces/ColumnAlteration.md
@@ -13,29 +13,9 @@ must be provided.

 ## Properties

-### dataType?
-
-```ts
-optional dataType: string;
-```
-
-A new data type for the column. If not provided then the data type will not be changed.
-Changing data types is limited to casting to the same general type. For example, these
-changes are valid:
-* `int32` -> `int64` (integers)
-* `double` -> `float` (floats)
-* `string` -> `large_string` (strings)
-But these changes are not:
-* `int32` -> `double` (mix integers and floats)
-* `string` -> `int32` (mix strings and integers)
-
-***
-
 ### nullable?

-```ts
-optional nullable: boolean;
-```
+> `optional` **nullable**: `boolean`

 Set the new nullability. Note that a nullable column cannot be made non-nullable.

@@ -43,9 +23,7 @@ Set the new nullability. Note that a nullable column cannot be made non-nullable

 ### path

-```ts
-path: string;
-```
+> **path**: `string`

 The path to the column to alter. This is a dot-separated path to the column.
 If it is a top-level column then it is just the name of the column. If it is
@@ -56,9 +34,7 @@ a nested column then it is the path to the column, e.g. "a.b.c" for a column

 ### rename?

-```ts
-optional rename: string;
-```
+> `optional` **rename**: `string`

 The new name of the column. If not provided then the name will not be changed.
 This must be distinct from the names of all other columns in the table.
--- a/docs/src/js/interfaces/CompactionStats.md
+++ b/docs/src/js/interfaces/CompactionStats.md
@@ -1,49 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / CompactionStats
-
-# Interface: CompactionStats
-
-Statistics about a compaction operation.
-
-## Properties
-
-### filesAdded
-
-```ts
-filesAdded: number;
-```
-
-The number of new, compacted data files added
-
-***
-
-### filesRemoved
-
-```ts
-filesRemoved: number;
-```
-
-The number of data files removed
-
-***
-
-### fragmentsAdded
-
-```ts
-fragmentsAdded: number;
-```
-
-The number of new, compacted fragments added
-
-***
-
-### fragmentsRemoved
-
-```ts
-fragmentsRemoved: number;
-```
-
-The number of fragments removed
--- a/docs/src/js/interfaces/ConnectionOptions.md
+++ b/docs/src/js/interfaces/ConnectionOptions.md
@@ -8,44 +8,9 @@

 ## Properties

-### apiKey?
-
-```ts
-optional apiKey: string;
-```
-
-(For LanceDB cloud only): the API key to use with LanceDB Cloud.
-
-Can also be set via the environment variable `LANCEDB_API_KEY`.
-
-***
-
-### clientConfig?
-
-```ts
-optional clientConfig: ClientConfig;
-```
-
-(For LanceDB cloud only): configuration for the remote HTTP client.
-
-***
-
-### hostOverride?
-
-```ts
-optional hostOverride: string;
-```
-
-(For LanceDB cloud only): the host to use for LanceDB cloud. Used
-for testing purposes.
-
-***
-
 ### readConsistencyInterval?

-```ts
-optional readConsistencyInterval: number;
-```
+> `optional` **readConsistencyInterval**: `number`

 (For LanceDB OSS only): The interval, in seconds, at which to check for
 updates to the table from other processes. If None, then consistency is not
@@ -59,22 +24,9 @@ always consistent.

 ***

-### region?
-
-```ts
-optional region: string;
-```
-
-(For LanceDB cloud only): the region to use for LanceDB cloud.
-Defaults to 'us-east-1'.
-
-***
-
 ### storageOptions?

-```ts
-optional storageOptions: Record<string, string>;
-```
+> `optional` **storageOptions**: `Record`&lt;`string`, `string`&gt;

 (For LanceDB OSS only): configuration for object storage.

--- a/docs/src/js/interfaces/CreateTableOptions.md
+++ b/docs/src/js/interfaces/CreateTableOptions.md
@@ -8,54 +8,15 @@

 ## Properties

-### ~~dataStorageVersion?~~
-
-```ts
-optional dataStorageVersion: string;
-```
-
-The version of the data storage format to use.
-
-The default is `stable`.
-Set to "legacy" to use the old format.
-
-#### Deprecated
-
-Pass `new_table_data_storage_version` to storageOptions instead.
-
-***
-
 ### embeddingFunction?

-```ts
-optional embeddingFunction: EmbeddingFunctionConfig;
-```
-
-***
-
-### ~~enableV2ManifestPaths?~~
-
-```ts
-optional enableV2ManifestPaths: boolean;
-```
-
-Use the new V2 manifest paths. These paths provide more efficient
-opening of datasets with many versions on object stores.  WARNING:
-turning this on will make the dataset unreadable for older versions
-of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
-use the LocalTable#migrateManifestPathsV2 method.
-
-#### Deprecated
-
-Pass `new_table_enable_v2_manifest_paths` to storageOptions instead.
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)

 ***

 ### existOk

-```ts
-existOk: boolean;
-```
+> **existOk**: `boolean`

 If this is true and the table already exists and the mode is "create"
 then no error will be raised.
@@ -64,9 +25,7 @@ then no error will be raised.

 ### mode

-```ts
-mode: "overwrite" | "create";
-```
+> **mode**: `"overwrite"` \| `"create"`

 The mode to use when creating the table.

@@ -80,17 +39,13 @@ If this is set to "overwrite" then any existing table will be replaced.

 ### schema?

-```ts
-optional schema: SchemaLike;
-```
+> `optional` **schema**: `SchemaLike`

 ***

 ### storageOptions?

-```ts
-optional storageOptions: Record<string, string>;
-```
+> `optional` **storageOptions**: `Record`&lt;`string`, `string`&gt;

 Configuration for object storage.

@@ -98,3 +53,13 @@ Options already set on the connection will be inherited by the table,
 but can be overridden here.

 The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+
+***
+
+### useLegacyFormat?
+
+> `optional` **useLegacyFormat**: `boolean`
+
+If true then data files will be written with the legacy format
+
+The default is true while the new format is in beta
--- a/docs/src/js/interfaces/FtsOptions.md
+++ b/docs/src/js/interfaces/FtsOptions.md
@@ -6,98 +6,20 @@

 # Interface: FtsOptions

-Options to create a full text search index
+Options to create an `FTS` index

 ## Properties

-### asciiFolding?
-
-```ts
-optional asciiFolding: boolean;
-```
-
-whether to remove punctuation
-
-***
-
-### baseTokenizer?
-
-```ts
-optional baseTokenizer: "raw" | "simple" | "whitespace";
-```
-
-The tokenizer to use when building the index.
-The default is "simple".
-
-The following tokenizers are available:
-
-"simple" - Simple tokenizer. This tokenizer splits the text into tokens using whitespace and punctuation as a delimiter.
-
-"whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
-
-"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
-
-***
-
-### language?
-
-```ts
-optional language: string;
-```
-
-language for stemming and stop words
-this is only used when `stem` or `remove_stop_words` is true
-
-***
-
-### lowercase?
-
-```ts
-optional lowercase: boolean;
-```
-
-whether to lowercase tokens
-
-***
-
-### maxTokenLength?
-
-```ts
-optional maxTokenLength: number;
-```
-
-maximum token length
-tokens longer than this length will be ignored
-
-***
-
-### removeStopWords?
-
-```ts
-optional removeStopWords: boolean;
-```
-
-whether to remove stop words
-
-***
-
-### stem?
-
-```ts
-optional stem: boolean;
-```
-
-whether to stem tokens
-
-***
-
 ### withPosition?

-```ts
-optional withPosition: boolean;
-```
+> `optional` **withPosition**: `boolean`

-Whether to build the index with positions.
-True by default.
-If set to false, the index will not store the positions of the tokens in the text,
-which will make the index smaller and faster to build, but will not support phrase queries.
+Whether to store the positions of the term in the document.
+
+If this is true then the index will store the positions of the term in the document.
+This allows phrase queries to be run. But it also increases the size of the index,
+and the time to build the index.
+
+The default value is true.
+
+***
--- a/docs/src/js/interfaces/FullTextSearchOptions.md
+++ b/docs/src/js/interfaces/FullTextSearchOptions.md
@@ -1,22 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / FullTextSearchOptions
-
-# Interface: FullTextSearchOptions
-
-Options that control the behavior of a full text search
-
-## Properties
-
-### columns?
-
-```ts
-optional columns: string | string[];
-```
-
-The columns to search
-
-If not specified, all indexed columns will be searched.
-For now, only one column can be searched.
--- a/docs/src/js/interfaces/HnswPqOptions.md
+++ b/docs/src/js/interfaces/HnswPqOptions.md
@@ -1,149 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / HnswPqOptions
-
-# Interface: HnswPqOptions
-
-Options to create an `HNSW_PQ` index
-
-## Properties
-
-### distanceType?
-
-```ts
-optional distanceType: "l2" | "cosine" | "dot";
-```
-
-The distance metric used to train the index.
-
-Default value is "l2".
-
-The following distance types are available:
-
-"l2" - Euclidean distance. This is a very common distance metric that
-accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
-
-"cosine" - Cosine distance.  Cosine distance is a distance metric
-calculated from the cosine similarity between two vectors. Cosine
-similarity is a measure of similarity between two non-zero vectors of an
-inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
-magnitude of the vectors.  Cosine distance has a range of [0, 2].
-
-"dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
-
-***
-
-### efConstruction?
-
-```ts
-optional efConstruction: number;
-```
-
-The number of candidates to evaluate during the construction of the HNSW graph.
-
-The default value is 300.
-
-This value controls the tradeoff between build speed and accuracy.
-The higher the value the more accurate the build but the slower it will be.
-150 to 300 is the typical range. 100 is a minimum for good quality search
-results. In most cases, there is no benefit to setting this higher than 500.
-This value should be set to a value that is not less than `ef` in the search phase.
-
-***
-
-### m?
-
-```ts
-optional m: number;
-```
-
-The number of neighbors to select for each vector in the HNSW graph.
-
-The default value is 20.
-
-This value controls the tradeoff between search speed and accuracy.
-The higher the value the more accurate the search but the slower it will be.
-
-***
-
-### maxIterations?
-
-```ts
-optional maxIterations: number;
-```
-
-Max iterations to train kmeans.
-
-The default value is 50.
-
-When training an IVF index we use kmeans to calculate the partitions.  This parameter
-controls how many iterations of kmeans to run.
-
-Increasing this might improve the quality of the index but in most cases the parameter
-is unused because kmeans will converge with fewer iterations.  The parameter is only
-used in cases where kmeans does not appear to converge.  In those cases it is unlikely
-that setting this larger will lead to the index converging anyways.
-
-***
-
-### numPartitions?
-
-```ts
-optional numPartitions: number;
-```
-
-The number of IVF partitions to create.
-
-For HNSW, we recommend a small number of partitions. Setting this to 1 works
-well for most tables. For very large tables, training just one HNSW graph
-will require too much memory. Each partition becomes its own HNSW graph, so
-setting this value higher reduces the peak memory use of training.
-
-***
-
-### numSubVectors?
-
-```ts
-optional numSubVectors: number;
-```
-
-Number of sub-vectors of PQ.
-
-This value controls how much the vector is compressed during the quantization step.
-The more sub vectors there are the less the vector is compressed.  The default is
-the dimension of the vector divided by 16.  If the dimension is not evenly divisible
-by 16 we use the dimension divded by 8.
-
-The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
-us to use efficient SIMD instructions.
-
-If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
-will likely result in poor performance.
-
-***
-
-### sampleRate?
-
-```ts
-optional sampleRate: number;
-```
-
-The rate used to calculate the number of training vectors for kmeans.
-
-Default value is 256.
-
-When an IVF index is trained, we need to calculate partitions.  These are groups
-of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-
-Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-random sample of the data.  This parameter controls the size of the sample.  The total
-number of vectors used to train the index is `sample_rate * num_partitions`.
-
-Increasing this value might improve the quality of the index but in most cases the
-default should be sufficient.
--- a/docs/src/js/interfaces/HnswSqOptions.md
+++ b/docs/src/js/interfaces/HnswSqOptions.md
@@ -1,128 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / HnswSqOptions
-
-# Interface: HnswSqOptions
-
-Options to create an `HNSW_SQ` index
-
-## Properties
-
-### distanceType?
-
-```ts
-optional distanceType: "l2" | "cosine" | "dot";
-```
-
-The distance metric used to train the index.
-
-Default value is "l2".
-
-The following distance types are available:
-
-"l2" - Euclidean distance. This is a very common distance metric that
-accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
-
-"cosine" - Cosine distance.  Cosine distance is a distance metric
-calculated from the cosine similarity between two vectors. Cosine
-similarity is a measure of similarity between two non-zero vectors of an
-inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
-magnitude of the vectors.  Cosine distance has a range of [0, 2].
-
-"dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
-
-***
-
-### efConstruction?
-
-```ts
-optional efConstruction: number;
-```
-
-The number of candidates to evaluate during the construction of the HNSW graph.
-
-The default value is 300.
-
-This value controls the tradeoff between build speed and accuracy.
-The higher the value the more accurate the build but the slower it will be.
-150 to 300 is the typical range. 100 is a minimum for good quality search
-results. In most cases, there is no benefit to setting this higher than 500.
-This value should be set to a value that is not less than `ef` in the search phase.
-
-***
-
-### m?
-
-```ts
-optional m: number;
-```
-
-The number of neighbors to select for each vector in the HNSW graph.
-
-The default value is 20.
-
-This value controls the tradeoff between search speed and accuracy.
-The higher the value the more accurate the search but the slower it will be.
-
-***
-
-### maxIterations?
-
-```ts
-optional maxIterations: number;
-```
-
-Max iterations to train kmeans.
-
-The default value is 50.
-
-When training an IVF index we use kmeans to calculate the partitions.  This parameter
-controls how many iterations of kmeans to run.
-
-Increasing this might improve the quality of the index but in most cases the parameter
-is unused because kmeans will converge with fewer iterations.  The parameter is only
-used in cases where kmeans does not appear to converge.  In those cases it is unlikely
-that setting this larger will lead to the index converging anyways.
-
-***
-
-### numPartitions?
-
-```ts
-optional numPartitions: number;
-```
-
-The number of IVF partitions to create.
-
-For HNSW, we recommend a small number of partitions. Setting this to 1 works
-well for most tables. For very large tables, training just one HNSW graph
-will require too much memory. Each partition becomes its own HNSW graph, so
-setting this value higher reduces the peak memory use of training.
-
-***
-
-### sampleRate?
-
-```ts
-optional sampleRate: number;
-```
-
-The rate used to calculate the number of training vectors for kmeans.
-
-Default value is 256.
-
-When an IVF index is trained, we need to calculate partitions.  These are groups
-of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-
-Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-random sample of the data.  This parameter controls the size of the sample.  The total
-number of vectors used to train the index is `sample_rate * num_partitions`.
-
-Increasing this value might improve the quality of the index but in most cases the
-default should be sufficient.
--- a/docs/src/js/interfaces/IndexConfig.md
+++ b/docs/src/js/interfaces/IndexConfig.md
@@ -12,9 +12,7 @@ A description of an index currently configured on a column

 ### columns

-```ts
-columns: string[];
-```
+> **columns**: `string`[]

 The columns in the index

@@ -25,9 +23,7 @@ be more columns to represent composite indices.

 ### indexType

-```ts
-indexType: string;
-```
+> **indexType**: `string`

 The type of the index

@@ -35,8 +31,6 @@ The type of the index

 ### name

-```ts
-name: string;
-```
+> **name**: `string`

 The name of the index
--- a/docs/src/js/interfaces/IndexMetadata.md
+++ b/docs/src/js/interfaces/IndexMetadata.md
@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexMetadata
+
+# Interface: IndexMetadata
+
+## Properties
+
+### indexType?
+
+> `optional` **indexType**: `string`
+
+***
+
+### metricType?
+
+> `optional` **metricType**: `string`
--- a/docs/src/js/interfaces/IndexOptions.md
+++ b/docs/src/js/interfaces/IndexOptions.md
@@ -10,9 +10,7 @@

 ### config?

-```ts
-optional config: Index;
-```
+> `optional` **config**: [`Index`](../classes/Index.md)

 Advanced index configuration

@@ -28,9 +26,7 @@ will be used to determine the most useful kind of index to create.

 ### replace?

-```ts
-optional replace: boolean;
-```
+> `optional` **replace**: `boolean`

 Whether to replace the existing index

--- a/docs/src/js/interfaces/IndexStatistics.md
+++ b/docs/src/js/interfaces/IndexStatistics.md
@@ -8,52 +8,32 @@

 ## Properties

-### distanceType?
+### indexType?

-```ts
-optional distanceType: string;
-```
-
-The type of the distance function used by the index. This is only
-present for vector indices. Scalar and full text search indices do
-not have a distance function.
-
-***
-
-### indexType
-
-```ts
-indexType: string;
-```
+> `optional` **indexType**: `string`

 The type of the index

 ***

+### indices
+
+> **indices**: [`IndexMetadata`](IndexMetadata.md)[]
+
+The metadata for each index
+
+***
+
 ### numIndexedRows

-```ts
-numIndexedRows: number;
-```
+> **numIndexedRows**: `number`

 The number of rows indexed by the index

 ***

-### numIndices?
-
-```ts
-optional numIndices: number;
-```
-
-The number of parts this index is split into.
-
-***
-
 ### numUnindexedRows

-```ts
-numUnindexedRows: number;
-```
+> **numUnindexedRows**: `number`

 The number of rows not indexed
--- a/docs/src/js/interfaces/IvfPqOptions.md
+++ b/docs/src/js/interfaces/IvfPqOptions.md
@@ -12,9 +12,7 @@ Options to create an `IVF_PQ` index

 ### distanceType?

-```ts
-optional distanceType: "l2" | "cosine" | "dot";
-```
+> `optional` **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`

 Distance type to use to build the index.

@@ -52,9 +50,7 @@ L2 norm is 1), then dot distance is equivalent to the cosine distance.

 ### maxIterations?

-```ts
-optional maxIterations: number;
-```
+> `optional` **maxIterations**: `number`

 Max iteration to train IVF kmeans.

@@ -68,26 +64,9 @@ The default value is 50.

 ***

-### numBits?
-
-```ts
-optional numBits: number;
-```
-
-Number of bits per sub-vector.
-
-This value controls how much each subvector is compressed.  The more bits the more
-accurate the index will be but the slower search.  The default is 8 bits.
-
-The number of bits must be 4 or 8.
-
-***
-
 ### numPartitions?

-```ts
-optional numPartitions: number;
-```
+> `optional` **numPartitions**: `number`

 The number of IVF partitions to create.

@@ -103,9 +82,7 @@ part of the search (searching within a partition) will be slow.

 ### numSubVectors?

-```ts
-optional numSubVectors: number;
-```
+> `optional` **numSubVectors**: `number`

 Number of sub-vectors of PQ.

@@ -124,9 +101,7 @@ will likely result in poor performance.

 ### sampleRate?

-```ts
-optional sampleRate: number;
-```
+> `optional` **sampleRate**: `number`

 The number of vectors, per partition, to sample when training IVF kmeans.

--- a/docs/src/js/interfaces/OpenTableOptions.md
+++ b/docs/src/js/interfaces/OpenTableOptions.md
@@ -1,40 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / OpenTableOptions
-
-# Interface: OpenTableOptions
-
-## Properties
-
-### indexCacheSize?
-
-```ts
-optional indexCacheSize: number;
-```
-
-Set the size of the index cache, specified as a number of entries
-
-The exact meaning of an "entry" will depend on the type of index:
- IVF: there is one entry for each IVF partition
- BTREE: there is one entry for the entire index
-
-This cache applies to the entire opened table, across all indices.
-Setting this value higher will increase performance on larger datasets
-at the expense of more RAM
-
-***
-
-### storageOptions?
-
-```ts
-optional storageOptions: Record<string, string>;
-```
-
-Configuration for object storage.
-
-Options already set on the connection will be inherited by the table,
-but can be overridden here.
-
-The available options are described at https://lancedb.github.io/lancedb/guides/storage/
--- a/docs/src/js/interfaces/OptimizeOptions.md
+++ b/docs/src/js/interfaces/OptimizeOptions.md
@@ -1,39 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / OptimizeOptions
-
-# Interface: OptimizeOptions
-
-## Properties
-
-### cleanupOlderThan
-
-```ts
-cleanupOlderThan: Date;
-```
-
-If set then all versions older than the given date
-be removed.  The current version will never be removed.
-The default is 7 days
-
-#### Example
-
-```ts
-// Delete all versions older than 1 day
-const olderThan = new Date();
-olderThan.setDate(olderThan.getDate() - 1));
-tbl.cleanupOlderVersions(olderThan);
-
-// Delete all versions except the current version
-tbl.cleanupOlderVersions(new Date());
-```
-
-***
-
-### deleteUnverified
-
-```ts
-deleteUnverified: boolean;
-```
--- a/docs/src/js/interfaces/OptimizeStats.md
+++ b/docs/src/js/interfaces/OptimizeStats.md
@@ -1,29 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / OptimizeStats
-
-# Interface: OptimizeStats
-
-Statistics about an optimize operation
-
-## Properties
-
-### compaction
-
-```ts
-compaction: CompactionStats;
-```
-
-Statistics about the compaction operation
-
-***
-
-### prune
-
-```ts
-prune: RemovalStats;
-```
-
-Statistics about the removal operation
--- a/docs/src/js/interfaces/QueryExecutionOptions.md
+++ b/docs/src/js/interfaces/QueryExecutionOptions.md
@@ -1,22 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / QueryExecutionOptions
-
-# Interface: QueryExecutionOptions
-
-Options that control the behavior of a particular query execution
-
-## Properties
-
-### maxBatchLength?
-
-```ts
-optional maxBatchLength: number;
-```
-
-The maximum number of rows to return in a single batch
-
-Batches may have fewer rows if the underlying data is stored
-in smaller chunks.
--- a/docs/src/js/interfaces/RemovalStats.md
+++ b/docs/src/js/interfaces/RemovalStats.md
@@ -1,29 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / RemovalStats
-
-# Interface: RemovalStats
-
-Statistics about a cleanup operation
-
-## Properties
-
-### bytesRemoved
-
-```ts
-bytesRemoved: number;
-```
-
-The number of bytes removed
-
-***
-
-### oldVersionsRemoved
-
-```ts
-oldVersionsRemoved: number;
-```
-
-The number of old versions removed
--- a/docs/src/js/interfaces/RetryConfig.md
+++ b/docs/src/js/interfaces/RetryConfig.md
@@ -1,90 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / RetryConfig
-
-# Interface: RetryConfig
-
-Retry configuration for the remote HTTP client.
-
-## Properties
-
-### backoffFactor?
-
-```ts
-optional backoffFactor: number;
-```
-
-The backoff factor to apply between retries. Default is 0.25. Between each retry
-the client will wait for the amount of seconds:
-`{backoff factor} * (2 ** ({number of previous retries}))`. So for the default
-of 0.25, the first retry will wait 0.25 seconds, the second retry will wait 0.5
-seconds, the third retry will wait 1 second, etc.
-
-You can also set this via the environment variable
-`LANCE_CLIENT_RETRY_BACKOFF_FACTOR`.
-
-***
-
-### backoffJitter?
-
-```ts
-optional backoffJitter: number;
-```
-
-The jitter to apply to the backoff factor, in seconds. Default is 0.25.
-
-A random value between 0 and `backoff_jitter` will be added to the backoff
-factor in seconds. So for the default of 0.25 seconds, between 0 and 250
-milliseconds will be added to the sleep between each retry.
-
-You can also set this via the environment variable
-`LANCE_CLIENT_RETRY_BACKOFF_JITTER`.
-
-***
-
-### connectRetries?
-
-```ts
-optional connectRetries: number;
-```
-
-The maximum number of retries for connection errors. Default is 3. You
-can also set this via the environment variable `LANCE_CLIENT_CONNECT_RETRIES`.
-
-***
-
-### readRetries?
-
-```ts
-optional readRetries: number;
-```
-
-The maximum number of retries for read errors. Default is 3. You can also
-set this via the environment variable `LANCE_CLIENT_READ_RETRIES`.
-
-***
-
-### retries?
-
-```ts
-optional retries: number;
-```
-
-The maximum number of retries for a request. Default is 3. You can also
-set this via the environment variable `LANCE_CLIENT_MAX_RETRIES`.
-
-***
-
-### statuses?
-
-```ts
-optional statuses: number[];
-```
-
-The HTTP status codes for which to retry the request. Default is
-[429, 500, 502, 503].
-
-You can also set this via the environment variable
-`LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.
--- a/docs/src/js/interfaces/TableNamesOptions.md
+++ b/docs/src/js/interfaces/TableNamesOptions.md
@@ -10,9 +10,7 @@

 ### limit?

-```ts
-optional limit: number;
-```
+> `optional` **limit**: `number`

 An optional limit to the number of results to return.

@@ -20,9 +18,7 @@ An optional limit to the number of results to return.

 ### startAfter?

-```ts
-optional startAfter: string;
-```
+> `optional` **startAfter**: `string`

 If present, only return names that come lexicographically after the
 supplied value.
--- a/docs/src/js/interfaces/TimeoutConfig.md
+++ b/docs/src/js/interfaces/TimeoutConfig.md
@@ -1,46 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / TimeoutConfig
-
-# Interface: TimeoutConfig
-
-Timeout configuration for remote HTTP client.
-
-## Properties
-
-### connectTimeout?
-
-```ts
-optional connectTimeout: number;
-```
-
-The timeout for establishing a connection in seconds. Default is 120
-seconds (2 minutes). This can also be set via the environment variable
-`LANCE_CLIENT_CONNECT_TIMEOUT`, as an integer number of seconds.
-
-***
-
-### poolIdleTimeout?
-
-```ts
-optional poolIdleTimeout: number;
-```
-
-The timeout for keeping idle connections in the connection pool in seconds.
-Default is 300 seconds (5 minutes). This can also be set via the
-environment variable `LANCE_CLIENT_CONNECTION_TIMEOUT`, as an integer
-number of seconds.
-
-***
-
-### readTimeout?
-
-```ts
-optional readTimeout: number;
-```
-
-The timeout for reading data from the server in seconds. Default is 300
-seconds (5 minutes). This can also be set via the environment variable
-`LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
--- a/docs/src/js/interfaces/UpdateOptions.md
+++ b/docs/src/js/interfaces/UpdateOptions.md
@@ -10,9 +10,7 @@

 ### where

-```ts
-where: string;
-```
+> **where**: `string`

 A filter that limits the scope of the update.

--- a/docs/src/js/interfaces/Version.md
+++ b/docs/src/js/interfaces/Version.md
@@ -1,31 +0,0 @@
-[**@lancedb/lancedb**](../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../globals.md) / Version
-
-# Interface: Version
-
-## Properties
-
-### metadata
-
-```ts
-metadata: Record<string, string>;
-```
-
-***
-
-### timestamp
-
-```ts
-timestamp: Date;
-```
-
-***
-
-### version
-
-```ts
-version: number;
-```
--- a/docs/src/js/interfaces/WriteOptions.md
+++ b/docs/src/js/interfaces/WriteOptions.md
@@ -0,0 +1,17 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteOptions
+
+# Interface: WriteOptions
+
+Write options when creating a Table.
+
+## Properties
+
+### mode?
+
+> `optional` **mode**: [`WriteMode`](../enumerations/WriteMode.md)
+
+Write mode for writing to a table.
--- a/docs/src/js/namespaces/embedding/README.md
+++ b/docs/src/js/namespaces/embedding/README.md
@@ -12,19 +12,15 @@

 - [EmbeddingFunction](classes/EmbeddingFunction.md)
 - [EmbeddingFunctionRegistry](classes/EmbeddingFunctionRegistry.md)
- [TextEmbeddingFunction](classes/TextEmbeddingFunction.md)
+- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)

 ### Interfaces

 - [EmbeddingFunctionConfig](interfaces/EmbeddingFunctionConfig.md)
- [EmbeddingFunctionConstructor](interfaces/EmbeddingFunctionConstructor.md)
- [EmbeddingFunctionCreate](interfaces/EmbeddingFunctionCreate.md)
- [FieldOptions](interfaces/FieldOptions.md)
- [FunctionOptions](interfaces/FunctionOptions.md)

 ### Type Aliases

- [CreateReturnType](type-aliases/CreateReturnType.md)
+- [OpenAIOptions](type-aliases/OpenAIOptions.md)

 ### Functions

--- a/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+++ b/docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
@@ -8,40 +8,21 @@

 An embedding function that automatically creates vector representation for a given column.

-It's important subclasses pass the **original** options to the super constructor
-and then pass those options to `resolveVariables` to resolve any variables before
-using them.
-
-## Example
-
-```ts
-class MyEmbeddingFunction extends EmbeddingFunction {
-  constructor(options: {model: string, timeout: number}) {
-    super(optionsRaw);
-    const options = this.resolveVariables(optionsRaw);
-    this.model = options.model;
-    this.timeout = options.timeout;
-  }
-}
-```
-
 ## Extended by

- [`TextEmbeddingFunction`](TextEmbeddingFunction.md)
+- [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)

 ## Type Parameters

 • **T** = `any`

-• **M** *extends* [`FunctionOptions`](../interfaces/FunctionOptions.md) = [`FunctionOptions`](../interfaces/FunctionOptions.md)
+• **M** *extends* `FunctionOptions` = `FunctionOptions`

 ## Constructors

 ### new EmbeddingFunction()

-```ts
-new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
-```
+> **new EmbeddingFunction**&lt;`T`, `M`&gt;(): [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`T`, `M`&gt;

 #### Returns

@@ -51,15 +32,13 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>

 ### computeQueryEmbeddings()

-```ts
-computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
-```
+> **computeQueryEmbeddings**(`data`): `Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;

 Compute the embeddings for a single query

 #### Parameters

-* **data**: `T`
+• **data**: `T`

 #### Returns

@@ -69,15 +48,13 @@ Compute the embeddings for a single query

 ### computeSourceEmbeddings()

-```ts
-abstract computeSourceEmbeddings(data): Promise<number[][] | Float32Array[] | Float64Array[]>
-```
+> `abstract` **computeSourceEmbeddings**(`data`): `Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;

 Creates a vector representation for the given values.

 #### Parameters

-* **data**: `T`[]
+• **data**: `T`[]

 #### Returns

@@ -87,9 +64,7 @@ Creates a vector representation for the given values.

 ### embeddingDataType()

-```ts
-abstract embeddingDataType(): Float<Floats>
-```
+> `abstract` **embeddingDataType**(): `Float`&lt;`Floats`&gt;

 The datatype of the embeddings

@@ -99,44 +74,9 @@ The datatype of the embeddings

 ***

-### getSensitiveKeys()
-
-```ts
-protected getSensitiveKeys(): string[]
-```
-
-Provide a list of keys in the function options that should be treated as
-sensitive. If users pass raw values for these keys, they will be rejected.
-
-#### Returns
-
-`string`[]
-
-***
-
-### init()?
-
-```ts
-optional init(): Promise<void>
-```
-
-Optionally load any resources needed for the embedding function.
-
-This method is called after the embedding function has been initialized
-but before any embeddings are computed. It is useful for loading local models
-or other resources that are needed for the embedding function to work.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-***
-
 ### ndims()

-```ts
-ndims(): undefined | number
-```
+> **ndims**(): `undefined` \| `number`

 The number of dimensions of the embeddings

@@ -146,79 +86,77 @@ The number of dimensions of the embeddings

 ***

-### resolveVariables()
-
-```ts
-protected resolveVariables(config): Partial<M>
-```
-
-Apply variables to the config.
-
-#### Parameters
-
-* **config**: `Partial`&lt;`M`&gt;
-
-#### Returns
-
-`Partial`&lt;`M`&gt;
-
-***
-
 ### sourceField()

-```ts
-sourceField(optionsOrDatatype): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
-```
+> **sourceField**(`optionsOrDatatype`): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]

 sourceField is used in combination with `LanceSchema` to provide a declarative data model

 #### Parameters

-* **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;[`FieldOptions`](../interfaces/FieldOptions.md)&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
-    The options for the field or the datatype
+• **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+
+The options for the field or the datatype

 #### Returns

-[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;]
+[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]

 #### See

-[LanceSchema](../functions/LanceSchema.md)
+lancedb.LanceSchema

 ***

 ### toJSON()

-```ts
-toJSON(): Record<string, any>
-```
+> `abstract` **toJSON**(): `Partial`&lt;`M`&gt;

-Get the original arguments to the constructor, to serialize them so they
-can be used to recreate the embedding function later.
+Convert the embedding function to a JSON object
+It is used to serialize the embedding function to the schema
+It's important that any object returned by this method contains all the necessary
+information to recreate the embedding function
+
+It should return the same object that was passed to the constructor
+If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly

 #### Returns

-`Record`&lt;`string`, `any`&gt;
+`Partial`&lt;`M`&gt;
+
+#### Example
+
+```ts
+class MyEmbeddingFunction extends EmbeddingFunction {
+  constructor(options: {model: string, timeout: number}) {
+    super();
+    this.model = options.model;
+    this.timeout = options.timeout;
+  }
+  toJSON() {
+    return {
+      model: this.model,
+      timeout: this.timeout,
+    };
+}
+```

 ***

 ### vectorField()

-```ts
-vectorField(optionsOrDatatype?): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
-```
+> **vectorField**(`optionsOrDatatype`?): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]

 vectorField is used in combination with `LanceSchema` to provide a declarative data model

 #### Parameters

-* **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;[`FieldOptions`](../interfaces/FieldOptions.md)&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
-    The options for the field
+• **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;

 #### Returns

-[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;]
+[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]

 #### See

-[LanceSchema](../functions/LanceSchema.md)
+lancedb.LanceSchema
--- a/docs/src/js/namespaces/embedding/classes/EmbeddingFunctionRegistry.md
+++ b/docs/src/js/namespaces/embedding/classes/EmbeddingFunctionRegistry.md
@@ -15,9 +15,7 @@ or TextEmbeddingFunction and registering it with the registry

 ### new EmbeddingFunctionRegistry()

-```ts
-new EmbeddingFunctionRegistry(): EmbeddingFunctionRegistry
-```
+> **new EmbeddingFunctionRegistry**(): [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)

 #### Returns

@@ -27,13 +25,11 @@ new EmbeddingFunctionRegistry(): EmbeddingFunctionRegistry

 ### functionToMetadata()

-```ts
-functionToMetadata(conf): Record<string, any>
-```
+> **functionToMetadata**(`conf`): `Record`&lt;`string`, `any`&gt;

 #### Parameters

-* **conf**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)
+• **conf**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)

 #### Returns

@@ -43,36 +39,35 @@ functionToMetadata(conf): Record<string, any>

 ### get()

-```ts
-get<T>(name): undefined | EmbeddingFunctionCreate<T>
-```
+> **get**&lt;`T`, `Name`&gt;(`name`): `Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`&lt;[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)&gt; : `undefined` \| `EmbeddingFunctionCreate`&lt;`T`&gt;

 Fetch an embedding function by name

 #### Type Parameters

-• **T** *extends* [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`unknown`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;
+• **T** *extends* [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`unknown`, `FunctionOptions`&gt;
+
+• **Name** *extends* `string` = `""`

 #### Parameters

-* **name**: `string`
-    The name of the function
+• **name**: `Name` *extends* `"openai"` ? `"openai"` : `string`
+
+The name of the function

 #### Returns

-`undefined` \| [`EmbeddingFunctionCreate`](../interfaces/EmbeddingFunctionCreate.md)&lt;`T`&gt;
+`Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`&lt;[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)&gt; : `undefined` \| `EmbeddingFunctionCreate`&lt;`T`&gt;

 ***

 ### getTableMetadata()

-```ts
-getTableMetadata(functions): Map<string, string>
-```
+> **getTableMetadata**(`functions`): `Map`&lt;`string`, `string`&gt;

 #### Parameters

-* **functions**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)[]
+• **functions**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)[]

 #### Returns

@@ -80,59 +75,21 @@ getTableMetadata(functions): Map<string, string>

 ***

-### getVar()
-
-```ts
-getVar(name): undefined | string
-```
-
-Get a variable.
-
-#### Parameters
-
-* **name**: `string`
-
-#### Returns
-
-`undefined` \| `string`
-
-#### See
-
-[setVar](EmbeddingFunctionRegistry.md#setvar)
-
-***
-
-### length()
-
-```ts
-length(): number
-```
-
-Get the number of registered functions
-
-#### Returns
-
-`number`
-
-***
-
 ### register()

-```ts
-register<T>(this, alias?): (ctor) => any
-```
+> **register**&lt;`T`&gt;(`this`, `alias`?): (`ctor`) => `any`

 Register an embedding function

 #### Type Parameters

-• **T** *extends* [`EmbeddingFunctionConstructor`](../interfaces/EmbeddingFunctionConstructor.md)&lt;[`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt; = [`EmbeddingFunctionConstructor`](../interfaces/EmbeddingFunctionConstructor.md)&lt;[`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;
+• **T** *extends* `EmbeddingFunctionConstructor`&lt;[`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt; = `EmbeddingFunctionConstructor`&lt;[`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;

 #### Parameters

-* **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)

-* **alias?**: `string`
+• **alias?**: `string`

 #### Returns

@@ -140,7 +97,7 @@ Register an embedding function

 ##### Parameters

-* **ctor**: `T`
+• **ctor**: `T`

 ##### Returns

@@ -154,43 +111,13 @@ Error if the function is already registered

 ### reset()

-```ts
-reset(this): void
-```
+> **reset**(`this`): `void`

 reset the registry to the initial state

 #### Parameters

-* **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
-
-#### Returns
-
-`void`
-
-***
-
-### setVar()
-
-```ts
-setVar(name, value): void
-```
-
-Set a variable. These can be accessed in the embedding function
-configuration using the syntax `$var:variable_name`. If they are not
-set, an error will be thrown letting you know which key is unset. If you
-want to supply a default value, you can add an additional part in the
-configuration like so: `$var:variable_name:default_value`. Default values
-can be used for runtime configurations that are not sensitive, such as
-whether to use a GPU for inference.
-
-The name must not contain colons. The default value can contain colons.
-
-#### Parameters
-
-* **name**: `string`
-
-* **value**: `string`
+• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)

 #### Returns

--- a/docs/src/js/namespaces/embedding/classes/OpenAIEmbeddingFunction.md
+++ b/docs/src/js/namespaces/embedding/classes/OpenAIEmbeddingFunction.md
@@ -0,0 +1,196 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / OpenAIEmbeddingFunction
+
+# Class: OpenAIEmbeddingFunction
+
+An embedding function that automatically creates vector representation for a given column.
+
+## Extends
+
+- [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`string`, `Partial`&lt;[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)&gt;&gt;
+
+## Constructors
+
+### new OpenAIEmbeddingFunction()
+
+> **new OpenAIEmbeddingFunction**(`options`): [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+
+#### Parameters
+
+• **options**: `Partial`&lt;[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)&gt; = `...`
+
+#### Returns
+
+[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`constructor`](EmbeddingFunction.md#constructors)
+
+## Methods
+
+### computeQueryEmbeddings()
+
+> **computeQueryEmbeddings**(`data`): `Promise`&lt;`number`[]&gt;
+
+Compute the embeddings for a single query
+
+#### Parameters
+
+• **data**: `string`
+
+#### Returns
+
+`Promise`&lt;`number`[]&gt;
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`computeQueryEmbeddings`](EmbeddingFunction.md#computequeryembeddings)
+
+***
+
+### computeSourceEmbeddings()
+
+> **computeSourceEmbeddings**(`data`): `Promise`&lt;`number`[][]&gt;
+
+Creates a vector representation for the given values.
+
+#### Parameters
+
+• **data**: `string`[]
+
+#### Returns
+
+`Promise`&lt;`number`[][]&gt;
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`computeSourceEmbeddings`](EmbeddingFunction.md#computesourceembeddings)
+
+***
+
+### embeddingDataType()
+
+> **embeddingDataType**(): `Float`&lt;`Floats`&gt;
+
+The datatype of the embeddings
+
+#### Returns
+
+`Float`&lt;`Floats`&gt;
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`embeddingDataType`](EmbeddingFunction.md#embeddingdatatype)
+
+***
+
+### ndims()
+
+> **ndims**(): `number`
+
+The number of dimensions of the embeddings
+
+#### Returns
+
+`number`
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`ndims`](EmbeddingFunction.md#ndims)
+
+***
+
+### sourceField()
+
+> **sourceField**(`optionsOrDatatype`): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+
+sourceField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+
+The options for the field or the datatype
+
+#### Returns
+
+[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+
+#### See
+
+lancedb.LanceSchema
+
+#### Inherited from
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`sourceField`](EmbeddingFunction.md#sourcefield)
+
+***
+
+### toJSON()
+
+> **toJSON**(): `object`
+
+Convert the embedding function to a JSON object
+It is used to serialize the embedding function to the schema
+It's important that any object returned by this method contains all the necessary
+information to recreate the embedding function
+
+It should return the same object that was passed to the constructor
+If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
+
+#### Returns
+
+`object`
+
+##### model
+
+> **model**: `string` & `object` \| `"text-embedding-ada-002"` \| `"text-embedding-3-small"` \| `"text-embedding-3-large"`
+
+#### Example
+
+```ts
+class MyEmbeddingFunction extends EmbeddingFunction {
+  constructor(options: {model: string, timeout: number}) {
+    super();
+    this.model = options.model;
+    this.timeout = options.timeout;
+  }
+  toJSON() {
+    return {
+      model: this.model,
+      timeout: this.timeout,
+    };
+}
+```
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`toJSON`](EmbeddingFunction.md#tojson)
+
+***
+
+### vectorField()
+
+> **vectorField**(`optionsOrDatatype`?): [`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+
+vectorField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;`FieldOptions`&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
+
+#### Returns
+
+[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]
+
+#### See
+
+lancedb.LanceSchema
+
+#### Inherited from
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`vectorField`](EmbeddingFunction.md#vectorfield)
--- a/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+++ b/docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
@@ -1,262 +0,0 @@
-[**@lancedb/lancedb**](../../../README.md) • **Docs**
-
-***
-
-[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / TextEmbeddingFunction
-
-# Class: `abstract` TextEmbeddingFunction&lt;M&gt;
-
-an abstract class for implementing embedding functions that take text as input
-
-## Extends
-
- [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`string`, `M`&gt;
-
-## Type Parameters
-
-• **M** *extends* [`FunctionOptions`](../interfaces/FunctionOptions.md) = [`FunctionOptions`](../interfaces/FunctionOptions.md)
-
-## Constructors
-
-### new TextEmbeddingFunction()
-
-```ts
-new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
-```
-
-#### Returns
-
-[`TextEmbeddingFunction`](TextEmbeddingFunction.md)&lt;`M`&gt;
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`constructor`](EmbeddingFunction.md#constructors)
-
-## Methods
-
-### computeQueryEmbeddings()
-
-```ts
-computeQueryEmbeddings(data): Promise<number[] | Float32Array | Float64Array>
-```
-
-Compute the embeddings for a single query
-
-#### Parameters
-
-* **data**: `string`
-
-#### Returns
-
-`Promise`&lt;`number`[] \| `Float32Array` \| `Float64Array`&gt;
-
-#### Overrides
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`computeQueryEmbeddings`](EmbeddingFunction.md#computequeryembeddings)
-
-***
-
-### computeSourceEmbeddings()
-
-```ts
-computeSourceEmbeddings(data): Promise<number[][] | Float32Array[] | Float64Array[]>
-```
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-* **data**: `string`[]
-
-#### Returns
-
-`Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;
-
-#### Overrides
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`computeSourceEmbeddings`](EmbeddingFunction.md#computesourceembeddings)
-
-***
-
-### embeddingDataType()
-
-```ts
-embeddingDataType(): Float<Floats>
-```
-
-The datatype of the embeddings
-
-#### Returns
-
-`Float`&lt;`Floats`&gt;
-
-#### Overrides
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`embeddingDataType`](EmbeddingFunction.md#embeddingdatatype)
-
-***
-
-### generateEmbeddings()
-
-```ts
-abstract generateEmbeddings(texts, ...args): Promise<number[][] | Float32Array[] | Float64Array[]>
-```
-
-#### Parameters
-
-* **texts**: `string`[]
-
-* ...**args**: `any`[]
-
-#### Returns
-
-`Promise`&lt;`number`[][] \| `Float32Array`[] \| `Float64Array`[]&gt;
-
-***
-
-### getSensitiveKeys()
-
-```ts
-protected getSensitiveKeys(): string[]
-```
-
-Provide a list of keys in the function options that should be treated as
-sensitive. If users pass raw values for these keys, they will be rejected.
-
-#### Returns
-
-`string`[]
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`getSensitiveKeys`](EmbeddingFunction.md#getsensitivekeys)
-
-***
-
-### init()?
-
-```ts
-optional init(): Promise<void>
-```
-
-Optionally load any resources needed for the embedding function.
-
-This method is called after the embedding function has been initialized
-but before any embeddings are computed. It is useful for loading local models
-or other resources that are needed for the embedding function to work.
-
-#### Returns
-
-`Promise`&lt;`void`&gt;
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`init`](EmbeddingFunction.md#init)
-
-***
-
-### ndims()
-
-```ts
-ndims(): undefined | number
-```
-
-The number of dimensions of the embeddings
-
-#### Returns
-
-`undefined` \| `number`
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`ndims`](EmbeddingFunction.md#ndims)
-
-***
-
-### resolveVariables()
-
-```ts
-protected resolveVariables(config): Partial<M>
-```
-
-Apply variables to the config.
-
-#### Parameters
-
-* **config**: `Partial`&lt;`M`&gt;
-
-#### Returns
-
-`Partial`&lt;`M`&gt;
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`resolveVariables`](EmbeddingFunction.md#resolvevariables)
-
-***
-
-### sourceField()
-
-```ts
-sourceField(): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
-```
-
-sourceField is used in combination with `LanceSchema` to provide a declarative data model
-
-#### Returns
-
-[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;]
-
-#### See
-
-[LanceSchema](../functions/LanceSchema.md)
-
-#### Overrides
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`sourceField`](EmbeddingFunction.md#sourcefield)
-
-***
-
-### toJSON()
-
-```ts
-toJSON(): Record<string, any>
-```
-
-Get the original arguments to the constructor, to serialize them so they
-can be used to recreate the embedding function later.
-
-#### Returns
-
-`Record`&lt;`string`, `any`&gt;
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`toJSON`](EmbeddingFunction.md#tojson)
-
-***
-
-### vectorField()
-
-```ts
-vectorField(optionsOrDatatype?): [DataType<Type, any>, Map<string, EmbeddingFunction<any, FunctionOptions>>]
-```
-
-vectorField is used in combination with `LanceSchema` to provide a declarative data model
-
-#### Parameters
-
-* **optionsOrDatatype?**: `DataType`&lt;`Type`, `any`&gt; \| `Partial`&lt;[`FieldOptions`](../interfaces/FieldOptions.md)&lt;`DataType`&lt;`Type`, `any`&gt;&gt;&gt;
-    The options for the field
-
-#### Returns
-
-[`DataType`&lt;`Type`, `any`&gt;, `Map`&lt;`string`, [`EmbeddingFunction`](EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;]
-
-#### See
-
-[LanceSchema](../functions/LanceSchema.md)
-
-#### Inherited from
-
-[`EmbeddingFunction`](EmbeddingFunction.md).[`vectorField`](EmbeddingFunction.md#vectorfield)
--- a/docs/src/js/namespaces/embedding/functions/LanceSchema.md
+++ b/docs/src/js/namespaces/embedding/functions/LanceSchema.md
@@ -6,15 +6,13 @@

 # Function: LanceSchema()

-```ts
-function LanceSchema(fields): Schema
-```
+> **LanceSchema**(`fields`): `Schema`

 Create a schema with embedding functions.

 ## Parameters

-* **fields**: `Record`&lt;`string`, `object` \| [`object`, `Map`&lt;`string`, [`EmbeddingFunction`](../classes/EmbeddingFunction.md)&lt;`any`, [`FunctionOptions`](../interfaces/FunctionOptions.md)&gt;&gt;]&gt;
+• **fields**: `Record`&lt;`string`, `object` \| [`object`, `Map`&lt;`string`, [`EmbeddingFunction`](../classes/EmbeddingFunction.md)&lt;`any`, `FunctionOptions`&gt;&gt;]&gt;

 ## Returns

--- a/docs/src/js/namespaces/embedding/functions/getRegistry.md
+++ b/docs/src/js/namespaces/embedding/functions/getRegistry.md
@@ -6,9 +6,7 @@

 # Function: getRegistry()

-```ts
-function getRegistry(): EmbeddingFunctionRegistry
-```
+> **getRegistry**(): [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)

 Utility function to get the global instance of the registry

--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`!function(){var e,t,n;e="9627b71b382d201",t=function(){Reo.init({clientID:"9627b71b382d201"})},(n=document.createElement("script")).src="https://static.reo.dev/"+e+"/reo.js",n.defer=!0,n.onload=t,document.head.appendChild(n)}();`
				`@@ -0,0 +1 @@`
				TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.