[python] Bump version: 0.6.0 → 0.6.1

This will eventually replace the remote table implementations in python
and node.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.17
+current_version = 0.4.11
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True
@@ -7,16 +7,6 @@ tag_name = v{new_version}
 
 [bumpversion:file:node/package.json]
 
-[bumpversion:file:nodejs/package.json]
-
-[bumpversion:file:nodejs/npm/darwin-x64/package.json]
-
-[bumpversion:file:nodejs/npm/darwin-arm64/package.json]
-
-[bumpversion:file:nodejs/npm/linux-x64-gnu/package.json]
-
-[bumpversion:file:nodejs/npm/linux-arm64-gnu/package.json]
-
 [bumpversion:file:rust/ffi/node/Cargo.toml]
 
 [bumpversion:file:rust/lancedb/Cargo.toml]

.github/workflows/build_linux_wheel/action.yml

@@ -14,10 +14,6 @@ inputs:
     # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
     required: false
     default: "false"
-  manylinux:
-    description: "The manylinux version to build for"
-    required: false
-    default: "2_17"
 runs:
   using: "composite"
   steps:
@@ -32,7 +28,7 @@ runs:
         command: build
         working-directory: python
         target: x86_64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
+        manylinux: "2_17"
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
@@ -47,7 +43,7 @@ runs:
         command: build
         working-directory: python
         target: aarch64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
+        manylinux: "2_24"
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e

.github/workflows/cargo-publish.yml

@@ -8,9 +8,6 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Up-to-date compilers needed for fp16kernels.
-  CC: gcc-12
-  CXX: g++-12
 
 jobs:
   build:

.github/workflows/docs.yml

@@ -24,14 +24,10 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install dependecies needed for ubuntu
-        run: |
-          sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default        
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/docs_test.yml

@@ -18,30 +18,22 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "ubuntu-latest"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
         python-version: 3.11
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
     - name: Build Python
       working-directory: docs/test
       run:
@@ -56,8 +48,8 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
-    timeout-minutes: 60
+    runs-on: "ubuntu-latest"
+    timeout-minutes: 45
     strategy:
       fail-fast: false
     steps:
@@ -66,8 +58,6 @@ jobs:
       with:
         fetch-depth: 0
         lfs: true
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
     - name: Set up Node
       uses: actions/setup-node@v4
       with:
@@ -75,7 +65,6 @@ jobs:
     - name: Install dependecies needed for ubuntu
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
     - name: Rust cache
       uses: swatinem/rust-cache@v2
     - name: Install node dependencies

.github/workflows/node.yml

@@ -20,11 +20,31 @@ env:
   # "1" means line tables only, which is useful for panic tracebacks.
   #
   # Use native CPU to accelerate tests if possible, especially for f16
-  # target-cpu=haswell fixes failing ci build
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: Lint
+      run: |
+        npm ci
+        npm run lint
   linux:
     name: Linux (Node ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -107,7 +127,6 @@ jobs:
       AWS_ENDPOINT: http://localhost:4566
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
-      ALLOW_HTTP: true
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/nodejs.yml

@@ -28,10 +28,6 @@ jobs:
       run:
         shell: bash
         working-directory: nodejs
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -53,7 +49,6 @@ jobs:
         cargo clippy --all --all-features -- -D warnings
         npm ci
         npm run lint
-        npm run chkformat
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -85,12 +80,7 @@ jobs:
       run: |
         npm ci
         npm run build
-    - name: Setup localstack
-      working-directory: .
-      run: docker compose up --detach --wait
     - name: Test
-      env:
-        S3_TEST: "1"
       run: npm run test
   macos:
     timeout-minutes: 30
@@ -121,3 +111,4 @@ jobs:
     - name: Test
       run: |
         npm run test
+

.github/workflows/npm-publish.yml

@@ -2,7 +2,7 @@ name: NPM Publish
 
 on:
   release:
-    types: [published]
+    types: [ published ]
 
 jobs:
   node:
@@ -19,7 +19,7 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: "npm"
+          cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -31,7 +31,7 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: node-package
           path: |
@@ -61,41 +61,12 @@ jobs:
       - name: Build MacOS native node modules
         run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-darwin-${{ matrix.config.arch }}
+          name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
 
-  nodejs-macos:
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build MacOS native nodejs modules
-        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-darwin-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -132,63 +103,12 @@ jobs:
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-linux-${{ matrix.config.arch }}
+          name: native-linux
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
-  nodejs-linux:
-    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-      # The generic files are the same in all distros so we just pick
-      # one to do the upload.
-      - name: Upload Generic Artifacts
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-dist
-          path: |
-            nodejs/dist/*
-            !nodejs/dist/*.node
-
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -216,60 +136,25 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-windows
+          name: native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  nodejs-windows:
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows
-          path: |
-            nodejs/dist/*.node
-
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: node-*
+      - uses: actions/download-artifact@v3
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: "https://registry.npmjs.org"
+          registry-url: 'https://registry.npmjs.org'
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
@@ -279,45 +164,6 @@ jobs:
             npm publish $filename
           done
 
-  release-nodejs:
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: nodejs-dist
-          path: nodejs/dist
-      - uses: actions/download-artifact@v4
-        name: Download arch-specific binaries
-        with:
-          pattern: nodejs-*
-          path: nodejs/nodejs-artifacts
-          merge-multiple: true
-      - name: Display structure of downloaded files
-        run: find .
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Install napi-rs
-        run: npm install -g @napi-rs/cli
-      - name: Prepare artifacts
-        run: npx napi artifacts -d nodejs-artifacts
-      - name: Display structure of staged files
-        run: find npm
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: npm publish --access public
-
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
@@ -332,18 +178,3 @@ jobs:
       - uses: ./.github/workflows/update_package_lock
         with:
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-
-  update-package-lock-nodejs:
-    needs: [release-nodejs]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -6,25 +6,13 @@ on:
 
 jobs:
   linux:
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     timeout-minutes: 60
     strategy:
       matrix:
         python-minor-version: ["8"]
-        config:
-          - platform: x86_64
-            manylinux: "2_17"
-            extra_args: ""
-          - platform: x86_64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-          - platform: aarch64
-            manylinux: "2_24"
-            extra_args: ""
-          # We don't build fp16 kernels for aarch64, because it uses 
-          # cross compilation image, which doesn't have a new enough compiler.
+        platform:
+          - x86_64
+          - aarch64
     runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
@@ -38,16 +26,13 @@ jobs:
       - uses: ./.github/workflows/build_linux_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip ${{ matrix.config.extra_args }}"
-          arm-build: ${{ matrix.config.platform == 'aarch64' }}
-          manylinux: ${{ matrix.config.manylinux }}
+          args: "--release --strip"
+          arm-build: ${{ matrix.platform == 'aarch64' }}
       - uses: ./.github/workflows/upload_wheel
         with:
           token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           repo: "pypi"
   mac:
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
     timeout-minutes: 60
     runs-on: ${{ matrix.config.runner }}
     strategy:
@@ -73,15 +58,13 @@ jobs:
       - uses: ./.github/workflows/build_mac_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
+          args: "--release --strip --target ${{ matrix.config.target }}"
       - uses: ./.github/workflows/upload_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}
           token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
           repo: "pypi"
   windows:
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
     timeout-minutes: 60
     runs-on: windows-latest
     strategy:

.github/workflows/python.yml

@@ -66,7 +66,7 @@ jobs:
       - name: Install
         run: |
           pip install -e .[tests,dev,embeddings]
-          pip install tantivy
+          pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
           pip install mlx
       - name: Doctest
         run: pytest --doctest-modules python/lancedb
@@ -99,8 +99,6 @@ jobs:
           workspaces: python
       - uses: ./.github/workflows/build_linux_wheel
       - uses: ./.github/workflows/run_tests
-        with:
-          integration: true
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
@@ -190,6 +188,6 @@ jobs:
         run: |
           pip install "pydantic<2"
           pip install -e .[tests]
-          pip install tantivy
+          pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
       - name: Run tests
-        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
+        run: pytest -m "not slow" -x -v --durations=30 python/tests

.github/workflows/remote-integration.yml

@@ -0,0 +1,37 @@
+name: LanceDb Cloud Integration Test
+
+on:
+  workflow_run:
+    workflows: [Rust]
+    types:
+      - completed
+
+env:
+  LANCEDB_PROJECT: ${{ secrets.LANCEDB_PROJECT }}
+  LANCEDB_API_KEY: ${{ secrets.LANCEDB_API_KEY }}
+  LANCEDB_REGION: ${{ secrets.LANCEDB_REGION }}
+
+jobs:
+  test:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: cargo build --all-features
+    - name: Run Integration test
+      run: cargo test --tests -- --ignored

.github/workflows/run_tests/action.yml

@@ -5,10 +5,6 @@ inputs:
   python-minor-version:
     required: true
     description: "8 9 10 11 12"
-  integration:
-    required: false
-    description: "Run integration tests"
-    default: "false"
 runs:
   using: "composite"
   steps:
@@ -16,16 +12,6 @@ runs:
       shell: bash
       run: |
         pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
-    - name: Setup localstack for integration tests
-      if: ${{ inputs.integration == 'true' }}
+    - name: pytest
       shell: bash
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: pytest (with integration)
-      shell: bash
-      if: ${{ inputs.integration == 'true' }}
       run: pytest -m "not slow" -x -v --durations=30 python/python/tests
-    - name: pytest (no integration tests)
-      shell: bash
-      if: ${{ inputs.integration != 'true' }}
-      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -31,10 +31,6 @@ jobs:
       run:
         shell: bash
         working-directory: rust
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -58,10 +54,6 @@ jobs:
       run:
         shell: bash
         working-directory: rust
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -76,9 +68,6 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
     - name: Build
       run: cargo build --all-features
-    - name: Start S3 integration test environment
-      working-directory: .
-      run: docker compose up --detach --wait
     - name: Run tests
       run: cargo test --all-features
     - name: Run examples
@@ -108,8 +97,7 @@ jobs:
       - name: Build
         run: cargo build --all-features
       - name: Run tests
-        # Run with everything except the integration tests.
-        run: cargo test --features remote,fp16kernels
+        run: cargo test --all-features
   windows:
     runs-on: windows-2022
     steps:

.github/workflows/update_package_lock_nodejs/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock_nodejs
-description: "Update nodejs's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./nodejs
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/update_package_lock_run_nodejs.yml

@@ -1,19 +0,0 @@
-name: Update NodeJs package-lock.json
-
-on:
-  workflow_dispatch:
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.gitignore

@@ -34,7 +34,6 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
-nodejs/lancedb/native*
 dist
 
 ## Rust

.pre-commit-config.yaml

@@ -10,9 +10,3 @@ repos:
     rev: v0.2.2
     hooks:
     - id: ruff
-- repo: https://github.com/pre-commit/mirrors-prettier
-  rev: v3.1.0
-  hooks:
-    - id: prettier
-      files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

Cargo.toml

@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.10.15", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.10.15" }
-lance-linalg = { "version" = "=0.10.15" }
-lance-testing = { "version" = "=0.10.15" }
+lance = { "version" = "=0.10.1", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.1" }
+lance-linalg = { "version" = "=0.10.1" }
+lance-testing = { "version" = "=0.10.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"
@@ -28,16 +28,13 @@ arrow-schema = "50.0"
 arrow-arith = "50.0"
 arrow-cast = "50.0"
 async-trait = "0"
-chrono = "0.4.35"
+chrono = "0.4.23"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
 object_store = "0.9.0"
-pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
-regex = "1.10"
-lazy_static = "1"

README.md

@@ -1,13 +1,13 @@
 <div align="center">
 <p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
+<img width="275" alt="LanceDB Logo" src="https://user-images.githubusercontent.com/917119/226205734-6063d87a-1ecc-45fe-85be-1dea6383a3d8.png">
 
-**Developer-friendly, database for multimodal AI**
+**Developer-friendly, serverless vector database for AI applications**
 
 <a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
 <a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/) 
+[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
 [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
 [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 

ci/build_linux_artifacts_nodejs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_nodejs
-docker build \
-    -t lancedb-nodejs-manylinux \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-nodejs-manylinux \
-    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (nodejs binaries).
-# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/lancedb
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd nodejs
-    echo "Building nodejs library for $1"
-    export RUST_TARGET=$1
-    npm run build-release
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/build_windows_artifacts_nodejs.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (nodejs binaries).
-# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/lancedb"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "nodejs"
-    Write-Host "Building nodejs library for $target"
-    $env:RUST_TARGET=$target
-    npm run build-release
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/manylinux_nodejs/Dockerfile

@@ -1,31 +0,0 @@
-# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an
-# environment with a very old glibc, so that we are compatible with a wide
-# range of linux distributions.
-ARG ARCH=x86_64
-
-FROM quay.io/pypa/manylinux2014_${ARCH}
-
-ARG ARCH=x86_64
-ARG DOCKER_USER=default_user
-
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
-# Protobuf is also installed as root.
-COPY install_protobuf.sh install_protobuf.sh
-RUN ./install_protobuf.sh ${ARCH}
-
-ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
-
-# We switch to the user to install Rust and Node, since those like to be
-# installed at the user level.
-USER ${DOCKER_USER}
-
-COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
-RUN cp /prepare_manylinux_node.sh $HOME/ && \
-    cd $HOME && \
-    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd nodejs
-npm ci
-npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1u \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# Installs protobuf compiler. Should be run as root.
-set -e
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=x86_64
-else
-    # gnu target
-    ARCH=aarch_64
-fi
-
-PB_REL=https://github.com/protocolbuffers/protobuf/releases
-PB_VERSION=23.1
-curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-install_node() {
-    echo "Installing node..."
-
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-
-    source "$HOME"/.bashrc
-
-    nvm install --no-progress 16
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-install_node
-install_rust

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:3.3
+    image: localstack/localstack:0.14
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb,kms
+      - SERVICES=s3,dynamodb
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
       interval: 5s
       retries: 3
       start_period: 10s

docs/mkdocs.yml

@@ -27,6 +27,7 @@ theme:
     - content.tabs.link
     - content.action.edit
     - toc.follow
+    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
@@ -38,46 +39,55 @@ theme:
   custom_dir: overrides
 
 plugins:
-  - search
-  - autorefs
-  - mkdocstrings:
+- search
+- autorefs
+- mkdocstrings:
     handlers:
       python:
         paths: [../python]
         options:
           docstring_style: numpy
-            heading_level: 3
+          heading_level: 4
           show_source: true
           show_symbol_type_in_heading: true
           show_signature_annotations: true
-            show_root_heading: true
           members_order: source
         import:
           # for cross references
           - https://arrow.apache.org/docs/objects.inv
           - https://pandas.pydata.org/docs/objects.inv
-  - mkdocs-jupyter
+- mkdocs-jupyter
+- ultralytics:
+    verbose: True
+    enabled: True
+    default_image: "assets/lancedb_and_lance.png" # Default image for all pages
+    add_image: True # Automatically add meta image
+    add_keywords: True # Add page keywords in the header tag
+    add_share_buttons: True # Add social share buttons
+    add_authors: False # Display page authors 
+    add_desc: False
+    add_dates: False
 
 markdown_extensions:
-  - admonition
-  - footnotes
-  - pymdownx.details
-  - pymdownx.highlight:
+- admonition
+- footnotes
+- pymdownx.details
+- pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
-  - pymdownx.inlinehilite
-  - pymdownx.snippets:
+- pymdownx.inlinehilite
+- pymdownx.snippets:
     base_path: ..
     dedent_subsections: true
-  - pymdownx.superfences
-  - pymdownx.tabbed:
+- pymdownx.superfences
+- pymdownx.tabbed:
     alternate_style: true
-  - md_in_html
-  - attr_list
+- md_in_html
+- attr_list
 
 nav:
-  - Home:
+- Home:
   - LanceDB: index.md
   - 🏃🏼‍♂️ Quick start: basic.md
   - 📚 Concepts:
@@ -94,18 +104,9 @@ nav:
       - Overview: hybrid_search/hybrid_search.md
       - Comparing Rerankers: hybrid_search/eval.md
       - Airbnb financial data example: notebooks/hybrid_search.ipynb
-          - Reranking:
-              - Quickstart: reranking/index.md
-              - Cohere Reranker: reranking/cohere.md
-              - Linear Combination Reranker: reranking/linear_combination.md
-              - Cross Encoder Reranker: reranking/cross_encoder.md
-              - ColBERT Reranker: reranking/colbert.md
-              - OpenAI Reranker: reranking/openai.md
-              - Building Custom Rerankers: reranking/custom_reranker.md
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
-          - Sync -> Async Migration Guide: migration.md
   - 🧬 Managing embeddings:
     - Overview: embeddings/index.md
     - Embedding functions: embeddings/embedding_functions.md
@@ -118,10 +119,9 @@ nav:
     - Pandas and PyArrow: python/pandas_and_pyarrow.md
     - Polars: python/polars_arrow.md
     - DuckDB: python/duckdb.md
-          - LangChain:
-            - LangChain 🔗: https://python.langchain.com/docs/integrations/vectorstores/lancedb/
-            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-          - LlamaIndex 🦙: https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
+    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
     - Pydantic: python/pydantic.md
     - Voxel51: integrations/voxel51.md
     - PromptTools: integrations/prompttools.md
@@ -140,27 +140,26 @@ nav:
       - Serverless Website Chatbot: examples/serverless_website_chatbot.md
       - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-          - 🦀 Rust:
-              - Overview: examples/examples_rust.md
+  - 🔧 CLI & Config: cli_config.md
   - 💭 FAQs: faq.md
   - ⚙️ API reference:
     - 🐍 Python: python/python.md
-          - 👾 JavaScript (vectordb): javascript/modules.md
-          - 👾 JavaScript (lancedb): javascript/modules.md
-          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
+    - 👾 JavaScript: javascript/modules.md
+    - 🦀 Rust: https://docs.rs/vectordb/latest/vectordb/
   - ☁️ LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:
       - 🐍 Python: python/saas-python.md
       - 👾 JavaScript: javascript/saas-modules.md
 
-  - Quick start: basic.md
-  - Concepts:
+
+- Quick start: basic.md
+- Concepts:
     - Vector search: concepts/vector_search.md
     - Indexing: concepts/index_ivfpq.md
     - Storage: concepts/storage.md
     - Data management: concepts/data_management.md
-  - Guides:
+- Guides:
     - Working with tables: guides/tables.md
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
@@ -169,53 +168,43 @@ nav:
       - Overview: hybrid_search/hybrid_search.md
       - Comparing Rerankers: hybrid_search/eval.md
       - Airbnb financial data example: notebooks/hybrid_search.ipynb
-      - Reranking:
-          - Quickstart: reranking/index.md
-          - Cohere Reranker: reranking/cohere.md
-          - Linear Combination Reranker: reranking/linear_combination.md
-          - Cross Encoder Reranker: reranking/cross_encoder.md
-          - ColBERT Reranker: reranking/colbert.md
-          - OpenAI Reranker: reranking/openai.md
-          - Building Custom Rerankers: reranking/custom_reranker.md
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
-      - Sync -> Async Migration Guide: migration.md
-  - Managing Embeddings:
+- Managing Embeddings:
   - Overview: embeddings/index.md
   - Embedding functions: embeddings/embedding_functions.md
   - Available models: embeddings/default_embedding_functions.md
   - User-defined embedding functions: embeddings/custom_embedding_function.md
   - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
   - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
-  - Integrations:
+- Integrations:
   - Overview: integrations/index.md
   - Pandas and PyArrow: python/pandas_and_pyarrow.md
   - Polars: python/polars_arrow.md
-      - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
-      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+  - DuckDB : python/duckdb.md
+  - LangChain 🦜️🔗↗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
   - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
   - PromptTools: integrations/prompttools.md
-  - Examples:
+- Python examples:
   - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
+- Javascript examples:
+  - Overview: examples/examples_js.md
+  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-  - API reference:
-      - Overview: api_reference.md
+- API reference:
   - Python: python/python.md
-      - Javascript (vectordb): javascript/modules.md
-      - Javascript (lancedb): js/modules.md
-      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
-  - LanceDB Cloud:
+  - Javascript: javascript/modules.md
+- LanceDB Cloud:
     - Overview: cloud/index.md
     - API reference:
       - 🐍 Python: python/saas-python.md
@@ -232,10 +221,3 @@ extra:
   analytics:
     provider: google
     property: G-B7NFM40W74
-  social:
-    - icon: fontawesome/brands/github
-      link: https://github.com/lancedb/lancedb
-    - icon: fontawesome/brands/x-twitter
-      link: https://twitter.com/lancedb
-    - icon: fontawesome/brands/linkedin
-      link: https://www.linkedin.com/company/lancedb

docs/requirements.txt

@@ -3,3 +3,4 @@ mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
 pydantic
+mkdocs-ultralytics-plugin==0.0.44

docs/src/ann_indexes.md

@@ -7,11 +7,20 @@ for brute-force scanning of the entire vector space.
 A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-## Disk-based Index
+Currently, LanceDB does _not_ automatically create the ANN index.
+LanceDB has optimized code for kNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
+If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
-Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
-the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
-See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
+In the future we will look to automatically create and configure the ANN index as data comes in.
+
+## Types of Index
+
+Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+
+- `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+  and then use **Product Quantization** to compress vectors in each partition.
+- `DiskANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+  represent the nearest neighbors of each vector.
 
 ## Creating an IVF_PQ Index
 
@@ -46,34 +55,12 @@ Lance supports `IVF_PQ` index type by default.
      --8<-- "docs/src/ann_indexes.ts:ingest"
      ```
 
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
-    ```
-
-    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
-
-The following IVF_PQ paramters can be specified:
-
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
+- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
   We also support "cosine" and "dot" distance as well.
-- **num_partitions**: The number of partitions in the index. The default is the square root
-  of the number of rows.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
-  a single PQ code. The default is the dimension of the vector divided by 16.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
+- **num_partitions** (default: 256): The number of partitions of the index.
+- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+  For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
+  a single PQ code.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -101,7 +88,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-=== "MacOS"
+=== "Macos"
 
      <!-- skip-test -->
      ```python
@@ -113,7 +100,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-Troubleshooting:
+Trouble shootings:
 
 If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
 PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
@@ -156,14 +143,6 @@ There are a couple of parameters that can be used to fine-tune the search:
      --8<-- "docs/src/ann_indexes.ts:search1"
      ```
 
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
-    ```
-
-    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
-
 The search will return the data requested in addition to the distance of each item.
 
 ### Filtering (where clause)
@@ -208,21 +187,13 @@ You can select the columns returned by the query using a select clause.
 
 ## FAQ
 
-### Why do I need to manually create an index?
-
-Currently, LanceDB does _not_ automatically create the ANN index.
-LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
-datasets of the order of ~100K vectors don't require index creation. If you can live with up to
-100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
-
 ### When is it necessary to create an ANN vector index?
 
-`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
-In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
-We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
-vector indices are usually not necessary.
+`LanceDB` has manually-tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
+For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
 
-For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
 
 ### How big is my index, and how many memory will it take?
 

docs/src/api_reference.md

@@ -1,8 +0,0 @@
-# API Reference
-
-The API reference for the LanceDB client SDKs are available at the following locations:
-
-- [Python](python/python.md)
-- [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/modules.md)
-- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -3,7 +3,7 @@
 !!! info "LanceDB can be run in a number of ways:"
 
     * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Directly from a client application like a Jupyter notebook for analytical workloads
+    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
     * Deployed as a remote serverless database
 
 ![](assets/lancedb_embedded_explanation.png)
@@ -24,11 +24,13 @@
 
 === "Rust"
 
+    !!! warning "Rust SDK is experimental, might introduce breaking changes in the near future"
+
     ```shell
-    cargo add lancedb
+    cargo add vectordb
     ```
 
-    !!! info "To use the lancedb create, you first need to install protobuf."
+    !!! info "To use the vectordb create, you first need to install protobuf."
 
     === "macOS"
 
@@ -42,27 +44,18 @@
         sudo apt install -y protobuf-compiler libssl-dev
         ```
 
-    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+    !!! info "Please also make sure you're using the same version of Arrow as in the [vectordb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
 
-## Connect to a database
+## How to connect to a database
 
 === "Python"
 
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:imports"
-    --8<-- "python/python/tests/docs/test_basic.py:connect"
-
-    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
+      import lancedb
+      uri = "data/sample-lancedb"
+      db = lancedb.connect(uri)
       ```
 
-    !!! note "Asynchronous Python API"
-
-        The asynchronous Python API is new and has some slight differences compared
-        to the synchronous API.  Feel free to start using the asynchronous version.
-        Once all features have migrated we will start to move the synchronous API to
-        use the same syntax as the asynchronous API.  To help with this migration we
-        have created a [migration guide](migration.md) detailing the differences.
-
 === "Typescript"
 
     ```typescript
@@ -71,44 +64,29 @@
     --8<-- "docs/src/basic_legacy.ts:open_db"
     ```
 
-    !!! note "`@lancedb/lancedb` vs. `vectordb`"
-
-        The Javascript SDK was originally released as `vectordb`.  In an effort to
-        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
-        API is being released as `lancedb`.  If you are starting new work we encourage
-        you to try out `lancedb`.  Once the new API is feature complete we will begin
-        slowly deprecating `vectordb` in favor of `lancedb`. There is a
-        [migration guide](migration.md) detailing the differences which will assist
-        you in this process.
-
 === "Rust"
 
     ```rust
     #[tokio::main]
     async fn main() -> Result<()> {
-        --8<-- "rust/lancedb/examples/simple.rs:connect"
+        --8<-- "rust/vectordb/examples/simple.rs:connect"
     }
     ```
 
-    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
+    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/vectordb/examples/simple.rs) for a full working example."
 
 LanceDB will create the directory if it doesn't exist (including parent directories).
 
 If you need a reminder of the uri, you can call `db.uri()`.
 
-## Create a table
-
-### Create a table from initial data
-
-If you have data to insert into the table at creation time, you can simultaneously create a
-table and insert the data into it. The schema of the data will be used as the schema of the
-table.
+## How to create a table
 
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
+    tbl = db.create_table("my_table",
+                    data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                          {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     If the table already exists, LanceDB will raise an error by default.
@@ -118,8 +96,10 @@ table.
     You can also pass in a pandas DataFrame directly:
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
+    import pandas as pd
+    df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    tbl = db.create_table("table_from_df", data=df)
     ```
 
 === "Typescript"
@@ -135,33 +115,27 @@ table.
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_table"
+    use arrow_schema::{DataType, Schema, Field};
+    use arrow_array::{RecordBatch, RecordBatchIterator};
+
+    --8<-- "rust/vectordb/examples/simple.rs:create_table"
     ```
 
-    If the table already exists, LanceDB will raise an error by default.  See
-    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
-    for details on how to overwrite (or open) existing tables instead.
+    If the table already exists, LanceDB will raise an error by default.
 
-    !!! Providing table records in Rust
+!!! info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
-        The Rust SDK currently expects data to be provided as an Arrow
-        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
-        Support for additional formats (such as serde or polars) is on the roadmap.
-
-!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
-
-### Create an empty table
+### Creating an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema, so that you can add
-data to the table at a later time (as long as it conforms to the schema). This is
-similar to a `CREATE TABLE` statement in SQL.
+In this case, you can create an empty table and specify the schema.
 
 === "Python"
 
       ```python
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+      import pyarrow as pa
+      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
+      tbl = db.create_table("empty_table", schema=schema)
       ```
 
 === "Typescript"
@@ -173,18 +147,17 @@ similar to a `CREATE TABLE` statement in SQL.
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
+    --8<-- "rust/vectordb/examples/simple.rs:create_empty_table"
     ```
 
-## Open an existing table
+## How to open an existing table
 
-Once created, you can open a table as follows:
+Once created, you can open a table using the following code:
 
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:open_table"
-    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
+    tbl = db.open_table("my_table")
     ```
 
 === "Typescript"
@@ -196,7 +169,7 @@ Once created, you can open a table as follows:
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
+    --8<-- "rust/vectordb/examples/simple.rs:open_with_existing_file"
     ```
 
 If you forget the name of your table, you can always get a listing of all table names:
@@ -204,8 +177,7 @@ If you forget the name of your table, you can always get a listing of all table
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:table_names"
-    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
+    print(db.table_names())
     ```
 
 === "Javascript"
@@ -217,18 +189,25 @@ If you forget the name of your table, you can always get a listing of all table
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:list_names"
+    --8<-- "rust/vectordb/examples/simple.rs:list_names"
     ```
 
-## Add data to a table
+## How to add data to a table
 
-After a table has been created, you can always add more data to it as follows:
+After a table has been created, you can always add more data to it using
 
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:add_data"
-    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+
+    # Option 1: Add a list of dicts to a table
+    data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
+    tbl.add(data)
+
+    # Option 2: Add a pandas DataFrame to a table
+    df = pd.DataFrame(data)
+    tbl.add(data)
     ```
 
 === "Typescript"
@@ -240,18 +219,17 @@ After a table has been created, you can always add more data to it as follows:
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:add"
+    --8<-- "rust/vectordb/examples/simple.rs:add"
     ```
 
-## Search for nearest neighbors
+## How to search for (approximate) nearest neighbors
 
-Once you've embedded the query, you can find its nearest neighbors as follows:
+Once you've embedded the query, you can find its nearest neighbors using the following code:
 
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
+    tbl.search([100, 100]).limit(2).to_pandas()
     ```
 
     This returns a pandas DataFrame with the results.
@@ -267,26 +245,16 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
     ```rust
     use futures::TryStreamExt;
 
-    --8<-- "rust/lancedb/examples/simple.rs:search"
+    --8<-- "rust/vectordb/examples/simple.rs:search"
     ```
 
-    !!! Query vectors in Rust
-        Rust does not yet support automatic execution of embedding functions.  You will need to
-        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
-        https://github.com/lancedb/lancedb/issues/994
-
-        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
-        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
-
 By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
 For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
-LanceDB allows you to create an ANN index on a table as follows:
 
 === "Python"
 
     ```py
-    --8<-- "python/python/tests/docs/test_basic.py:create_index"
-    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
+    tbl.create_index()
     ```
 
 === "Typescript"
@@ -298,17 +266,12 @@ LanceDB allows you to create an ANN index on a table as follows:
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_index"
+     --8<-- "rust/vectordb/examples/simple.rs:create_index"
     ```
 
-!!! note "Why do I need to create an index manually?"
-LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
-for really fast retrievals via a disk-based index, and the second is that data and query workloads can
-be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
-to fine-tune index size, query latency and accuracy. See the section on
-[ANN indexes](ann_indexes.md) for more details.
+Check [Approximate Nearest Neighbor (ANN) Indexes](/ann_indices.md) section for more details.
 
-## Delete rows from a table
+## How to delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
 which rows to delete, provide a filter that matches on the metadata columns.
@@ -317,8 +280,7 @@ This can delete any number of rows that match the filter.
 === "Python"
 
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
+    tbl.delete('item = "fizz"')
     ```
 
 === "Typescript"
@@ -330,13 +292,12 @@ This can delete any number of rows that match the filter.
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:delete"
+    --8<-- "rust/vectordb/examples/simple.rs:delete"
     ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause (`only_if()` in Rust) on a search. They can be as
-simple or complex as needed. To see what expressions are supported, see the
-[SQL filters](sql.md) section.
+as the `where()` clause on a search. They can be as simple or complex as needed.
+To see what expressions are supported, see the [SQL filters](sql.md) section.
 
 === "Python"
 
@@ -346,19 +307,14 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-=== "Rust"
-
-      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
-
-## Drop a table
+## How to remove a table
 
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
       ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      db.drop_table("my_table")
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
@@ -377,7 +333,7 @@ Use the `drop_table()` method on the database to remove a table.
 === "Rust"
 
     ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
+    --8<-- "rust/vectordb/examples/simple.rs:drop_table"
     ```
 
 !!! note "Bundling `vectordb` apps with Webpack"

docs/src/cli_config.md

@@ -0,0 +1,51 @@
+
+# CLI & Config
+
+## LanceDB CLI
+Once lanceDB is installed, you can access the CLI using `lancedb` command on the console.
+
+```
+lancedb
+```
+
+This lists out all the various command-line options available. You can get the usage or help for a particular command.
+
+```
+lancedb {command} --help
+```
+
+## LanceDB config
+LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
+To view your config settings, you can use:
+
+```
+lancedb config
+```
+
+These config parameters can be tuned using the cli.
+
+```
+lancedb {config_name} --{argument}
+```
+
+## LanceDB Opt-in Diagnostics
+When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
+These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
+
+### Get usage help
+
+```
+lancedb diagnostics --help
+```
+
+### Disable diagnostics
+
+```
+lancedb diagnostics --disabled
+```
+
+### Enable diagnostics
+
+```
+lancedb diagnostics --enabled
+```

docs/src/concepts/index_ivfpq.md

@@ -31,7 +31,7 @@ As an example, consider starting with 128-dimensional vector consisting of 32-bi
 
 While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
 
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
 
 ![](../assets/ivfpq_ivf_desc.webp)
 
@@ -81,4 +81,24 @@ The above query will perform a search on the table `tbl` using the given query v
 * `to_pandas()`: Convert the results to a pandas DataFrame
 
 And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.
+
+
+## FAQ
+
+### When is it necessary to create a vector index?
+
+LanceDB has manually-tuned SIMD code for computing vector distances. In our benchmarks, computing 100K pairs of 1K dimension vectors takes **<20ms**. For small datasets (<100K rows) or applications that can accept up to 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how much memory will it take?
+
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with 1024-dimension vectors, if we choose `num_sub_vectors = 64`, each sub-vector has `1024 / 64 = 16` float32 numbers. Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for IVF_PQ index?
+
+`num_partitions` is used to decide how many partitions the first level IVF index uses. Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train. On SIFT-1M dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency/recall.
+
+`num_sub_vectors` specifies how many PQ short codes to generate on each vector. Because PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/embeddings/default_embedding_functions.md

@@ -19,218 +19,49 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
 
 
-??? "Check out available sentence-transformer models here!"
-    ```markdown
-    - sentence-transformers/all-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-mpnet-base-v2
-    - sentence-transformers/gtr-t5-base
-    - sentence-transformers/LaBSE
-    - sentence-transformers/all-MiniLM-L6-v2
-    - sentence-transformers/bert-base-nli-max-tokens
-    - sentence-transformers/bert-base-nli-mean-tokens
-    - sentence-transformers/bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
-    - sentence-transformers/bert-large-nli-cls-token
-    - sentence-transformers/bert-large-nli-max-tokens
-    - sentence-transformers/bert-large-nli-mean-tokens
-    - sentence-transformers/bert-large-nli-stsb-mean-tokens
-    - sentence-transformers/distilbert-base-nli-max-tokens
-    - sentence-transformers/distilbert-base-nli-mean-tokens
-    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
-    - sentence-transformers/distilroberta-base-msmarco-v1
-    - sentence-transformers/distilroberta-base-msmarco-v2
-    - sentence-transformers/nli-bert-base-cls-pooling
-    - sentence-transformers/nli-bert-base-max-pooling
-    - sentence-transformers/nli-bert-base
-    - sentence-transformers/nli-bert-large-cls-pooling
-    - sentence-transformers/nli-bert-large-max-pooling
-    - sentence-transformers/nli-bert-large
-    - sentence-transformers/nli-distilbert-base-max-pooling
-    - sentence-transformers/nli-distilbert-base
-    - sentence-transformers/nli-roberta-base
-    - sentence-transformers/nli-roberta-large
-    - sentence-transformers/roberta-base-nli-mean-tokens
-    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
-    - sentence-transformers/roberta-large-nli-mean-tokens
-    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
-    - sentence-transformers/stsb-bert-base
-    - sentence-transformers/stsb-bert-large
-    - sentence-transformers/stsb-distilbert-base
-    - sentence-transformers/stsb-roberta-base
-    - sentence-transformers/stsb-roberta-large
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
-    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
-    - sentence-transformers/bert-base-nli-cls-token
-    - sentence-transformers/all-distilroberta-v1
-    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
-    - sentence-transformers/multi-qa-distilbert-cos-v1
-    - sentence-transformers/multi-qa-distilbert-dot-v1
-    - sentence-transformers/multi-qa-mpnet-base-cos-v1
-    - sentence-transformers/multi-qa-mpnet-base-dot-v1
-    - sentence-transformers/nli-distilroberta-base-v2
-    - sentence-transformers/all-MiniLM-L6-v1
-    - sentence-transformers/all-mpnet-base-v1
-    - sentence-transformers/all-mpnet-base-v2
-    - sentence-transformers/all-roberta-large-v1
-    - sentence-transformers/allenai-specter
-    - sentence-transformers/average_word_embeddings_glove.6B.300d
-    - sentence-transformers/average_word_embeddings_glove.840B.300d
-    - sentence-transformers/average_word_embeddings_komninos
-    - sentence-transformers/average_word_embeddings_levy_dependency
-    - sentence-transformers/clip-ViT-B-32-multilingual-v1
-    - sentence-transformers/clip-ViT-B-32
-    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
-    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
-    - sentence-transformers/distilroberta-base-paraphrase-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v2
-    - sentence-transformers/distiluse-base-multilingual-cased
-    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
-    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
-    - sentence-transformers/gtr-t5-large
-    - sentence-transformers/gtr-t5-xl
-    - sentence-transformers/gtr-t5-xxl
-    - sentence-transformers/msmarco-MiniLM-L-12-v3
-    - sentence-transformers/msmarco-MiniLM-L-6-v3
-    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
-    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
-    - sentence-transformers/msmarco-bert-base-dot-v5
-    - sentence-transformers/msmarco-bert-co-condensor
-    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
-    - sentence-transformers/msmarco-distilbert-base-tas-b
-    - sentence-transformers/msmarco-distilbert-base-v2
-    - sentence-transformers/msmarco-distilbert-base-v3
-    - sentence-transformers/msmarco-distilbert-base-v4
-    - sentence-transformers/msmarco-distilbert-cos-v5
-    - sentence-transformers/msmarco-distilbert-dot-v5
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
-    - sentence-transformers/msmarco-distilroberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-ance-firstp
-    - sentence-transformers/msmarco-roberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-v3
-    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    - sentence-transformers/nli-mpnet-base-v2
-    - sentence-transformers/nli-roberta-base-v2
-    - sentence-transformers/nq-distilbert-base-v1
-    - sentence-transformers/paraphrase-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-MiniLM-L3-v2
-    - sentence-transformers/paraphrase-MiniLM-L6-v2
-    - sentence-transformers/paraphrase-TinyBERT-L6-v2
-    - sentence-transformers/paraphrase-albert-base-v2
-    - sentence-transformers/paraphrase-albert-small-v2
-    - sentence-transformers/paraphrase-distilroberta-base-v1
-    - sentence-transformers/paraphrase-distilroberta-base-v2
-    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
-    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
-    - sentence-transformers/quora-distilbert-base
-    - sentence-transformers/quora-distilbert-multilingual
-    - sentence-transformers/sentence-t5-base
-    - sentence-transformers/sentence-t5-large
-    - sentence-transformers/sentence-t5-xxl
-    - sentence-transformers/sentence-t5-xl
-    - sentence-transformers/stsb-distilroberta-base-v2
-    - sentence-transformers/stsb-mpnet-base-v2
-    - sentence-transformers/stsb-roberta-base-v2
-    - sentence-transformers/stsb-xlm-r-multilingual
-    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
-    - sentence-transformers/clip-ViT-L-14
-    - sentence-transformers/clip-ViT-B-16
-    - sentence-transformers/use-cmlm-multilingual
-    - sentence-transformers/all-MiniLM-L12-v1
-    ```
+```python
+db = lancedb.connect("/tmp/db")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("sentence-transformers").create(device="cpu")
 
-!!! info
-    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
-    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
 
-!!! note "BAAI Embeddings example"
-    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
-    ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
-    db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-    class Words(LanceModel):
-        text: str = model.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
-
-    table = db.create_table("words", schema=Words)
-    table.add(
+table = db.create_table("words", schema=Words)
+table.add(
     [
-            {"text": "hello world"},
+        {"text": "hello world"}
         {"text": "goodbye world"}
     ]
-    )
+)
 
-    query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
-    print(actual.text)
-    ```
-Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
-
-
-### Huggingface embedding models
-We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
-
-Example usage - 
-```python
-import lancedb
-import pandas as pd
-
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-model = get_registry().get("huggingface").create(name='facebook/bart-base')
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
-table = db.create_table("greets", schema=Words)
-table.add()
-query = "old greeting"
+query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
 
-
 ### OpenAI embeddings
 LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
 
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
-| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
 
 
 ```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
 db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("openai").create()
 
 class Words(LanceModel):
     text: str = func.SourceField()
     vector: Vector(func.ndims()) = func.VectorField()
 
-table = db.create_table("words", schema=Words, mode="overwrite")
+table = db.create_table("words", schema=Words)
 table.add(
     [
-        {"text": "hello world"},
+        {"text": "hello world"}
         {"text": "goodbye world"}
     ]
     )
@@ -344,8 +175,7 @@ Supported Embedding modelIDs are:
 * `cohere.embed-english-v3`
 * `cohere.embed-multilingual-v3`
 
-Supported parameters (to be passed in `create` method) are:
-
+Supported paramters (to be passed in `create` method) are:
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
@@ -359,10 +189,6 @@ Supported parameters (to be passed in `create` method) are:
 Usage Example:
 
 ```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
 model = get_registry().get("bedrock-text").create()
 
 class TextModel(LanceModel):
@@ -396,13 +222,12 @@ This embedding function supports ingesting images as both bytes and urls. You ca
 !!! info
     LanceDB supports ingesting images directly from accessible links.
 
+
 ```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
 
 db = lancedb.connect(tmp_path)
-func = get_registry.get("open-clip").create()
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("open-clip").create()
 
 class Images(LanceModel):
     label: str
@@ -463,70 +288,4 @@ print(actual.label)
 
 ```
 
-### Imagebind embeddings
-We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
-
-This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"imagebind_huge"` | Name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
-
-Below is an example demonstrating how the API works:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry.get("imagebind").create()
-
-class ImageBindModel(LanceModel):
-    text: str
-    image_uri: str = func.SourceField()
-    audio_path: str
-    vector: Vector(func.ndims()) = func.VectorField()
-
-# add locally accessible image paths
-text_list=["A dog.", "A car", "A bird"]
-image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
-audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
-
-# Load data
-inputs = [
-    {"text": a, "audio_path": b, "image_uri": c}
-    for a, b, c in zip(text_list, audio_paths, image_paths)
-]
-
-#create table and add data
-table = db.create_table("img_bind", schema=ImageBindModel)
-table.add(inputs)
-```
-
-Now, we can search using any modality:
-
-#### image search
-```python
-query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
-actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "dog")
-```
-#### audio search
-
-```python
-query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
-actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "car")
-```
-#### Text search
-You can add any input query and fetch the result as follows:
-```python
-query = "an animal which flies and tweets" 
-actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "bird")
-```
-
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/index.md

@@ -12,63 +12,3 @@ LanceDB supports 3 methods of working with embeddings.
 
 For python users, there is also a legacy [with_embeddings API](./legacy.md).
 It is retained for compatibility and will be removed in a future version.
-
-## Quickstart
-
-To get started with embeddings, you can use the built-in embedding functions.
-
-### OpenAI Embedding function
-LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-)
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/examples/examples_rust.md

@@ -1,3 +0,0 @@
-# Examples: Rust
-
-Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -43,7 +43,7 @@ pip install lancedb
 We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
 
 ```
-pip install tantivy
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
 ```
 
 Create a new Python file and add the following code:

docs/src/examples/index.md

@@ -2,11 +2,10 @@
 
 ## Recipes and example code
 
-LanceDB provides language APIs, allowing you to embed a database in your language of choice.
+LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
 
 * 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](examples_js.md) examples
-* 🦀 Rust examples (coming soon)
+* 👾 [JavaScript](exampled_js.md) examples
 
 ## Applications powered by LanceDB
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -1,79 +1,11 @@
-// Creates an SVG robot icon (from Lucide)
-function robotSVG() {
-  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
-  svg.setAttribute("width", "24");
-  svg.setAttribute("height", "24");
-  svg.setAttribute("viewBox", "0 0 24 24");
-  svg.setAttribute("fill", "none");
-  svg.setAttribute("stroke", "currentColor");
-  svg.setAttribute("stroke-width", "2");
-  svg.setAttribute("stroke-linecap", "round");
-  svg.setAttribute("stroke-linejoin", "round");
-  svg.setAttribute("class", "lucide lucide-bot-message-square");
-
-  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path1.setAttribute("d", "M12 6V2H8");
-  svg.appendChild(path1);
-
-  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
-  svg.appendChild(path2);
-
-  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path3.setAttribute("d", "M2 12h2");
-  svg.appendChild(path3);
-
-  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path4.setAttribute("d", "M9 11v2");
-  svg.appendChild(path4);
-
-  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path5.setAttribute("d", "M15 11v2");
-  svg.appendChild(path5);
-
-  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path6.setAttribute("d", "M20 12h2");
-  svg.appendChild(path6);
-
-  return svg
-}
-
-// Creates the Fluidic Chatbot buttom
-function fluidicButton() {
-  var btn = document.createElement("a");
-  btn.href = "https://asklancedb.com";
-  btn.target = "_blank";
-  btn.style.position = "fixed";
-  btn.style.fontWeight = "bold";
-  btn.style.fontSize = ".8rem";
-  btn.style.right = "10px";
-  btn.style.bottom = "10px";
-  btn.style.width = "80px";
-  btn.style.height = "80px";
-  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
-  btn.style.color = "white";
-  btn.style.borderRadius = "5px";
-  btn.style.display = "flex";
-  btn.style.flexDirection = "column";
-  btn.style.justifyContent = "center";
-  btn.style.alignItems = "center";
-  btn.style.zIndex = "1000";
-  btn.style.opacity = "0";
-  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
-  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
-
-  setTimeout(function() {
-      btn.style.opacity = "1";
-      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
-  }, 0);
-
-  return btn
-}
-
-document.addEventListener("DOMContentLoaded", function() {
-  var btn = fluidicButton()
-  btn.appendChild(robotSVG());
-  var text = document.createTextNode("Ask AI");
-  btn.appendChild(text);
-  document.body.appendChild(btn);
-});
+document.addEventListener("DOMContentLoaded", function () {
+    var script = document.createElement("script");
+    script.src = "https://widget.kapa.ai/kapa-widget.bundle.js";
+    script.setAttribute("data-website-id", "c5881fae-cec0-490b-b45e-d83d131d4f25");
+    script.setAttribute("data-project-name", "LanceDB");
+    script.setAttribute("data-project-color", "#000000");
+    script.setAttribute("data-project-logo", "https://avatars.githubusercontent.com/u/108903835?s=200&v=4");
+  script.setAttribute("data-modal-example-questions","Help me create an IVF_PQ index,How do I do an exhaustive search?,How do I create a LanceDB table?,Can I use my own embedding function?");
+    script.async = true;
+    document.head.appendChild(script);
+  });

docs/src/faq.md

@@ -16,7 +16,7 @@ As we mention in our talk titled “[Lance, a modern columnar data format](https
 
 ### Why build in Rust? 🦀
 
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
 
 ### What is the difference between LanceDB OSS and LanceDB Cloud?
 
@@ -40,11 +40,11 @@ LanceDB and its underlying data format, Lance, are built to scale to really larg
 
 No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
 
-For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
 
 ### Does LanceDB support full-text search?
 
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
 
 ### How can I speed up data inserts?
 

docs/src/fts.md

@@ -1,6 +1,6 @@
 # Full-text search
 
-LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
 
 A hybrid search solution combining vector and full-text search is also on the way.
 
@@ -75,70 +75,21 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
-## Sorting
+## Syntax
 
-You can pre-sort the documents by specifying `ordering_field_names` when
-creating the full-text search index. Once pre-sorted, you can then specify
-`ordering_field_name` while searching to return results sorted by the given
-field. For example, 
+For full-text search you can perform either a phrase query like "the old man and the sea", 
+or a structured search query like "(Old AND Man) AND Sea".
+Double quotes are used to disambiguate.
 
-```
-table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
+For example: 
 
-(table.search("terms", ordering_field_name="sort_by_field")
- .limit(20)
- .to_list())
-```
+If you intended "they could have been dogs OR cats" as a phrase query, this actually
+raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
+this avoids the syntax error. However, it is cumbersome to have to remember what will
+conflict with the query syntax. Instead, if you search using 
+`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
+checking inside the quotes.
 
-!!! note
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see 
-    an error during indexing that looks like 
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
-## Phrase queries vs. terms queries
-
-For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
-query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
-
-!!! tip "Note"
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
-
-    ```py
-    # This raises a syntax error
-    table.search("they could have been dogs OR cats")
-    ```
-
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
-    the query is treated as a phrase query.
-
-    ```py
-    # This works!
-    table.search("they could have been dogs or cats")
-    ```
-
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
 
 ## Configurations
 
@@ -161,3 +112,4 @@ table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
 2. We currently only support local filesystem paths for the FTS index. 
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
+

docs/src/guides/storage.md

@@ -55,139 +55,18 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
     const db = await lancedb.connect("az://bucket/path");
     ```
 
-In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
+In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided using environment variables. In general, these environment variables are the same as those used by the respective cloud SDKs. The sections below describe the environment variables that can be used to configure each object store.
 
-```bash
-export TIMEOUT=60s
-```
-
-!!! note "`storage_options` availability"
-
-    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
-    It is not yet supported in the Python synchronous API.
-
-If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={"timeout": "60s"}
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path",
-                                     {storageOptions: {timeout: "60s"}});
-    ```
-
-Getting even more specific, you can set the `timeout` for only a particular table:
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async("s3://bucket/path")
-    table = await db.create_table(
-        "table",
-        [{"a": 1, "b": 2}],
-        storage_options={"timeout": "60s"}
-    )
-    ```
-
-=== "JavaScript"
-
-    <!-- skip-test -->
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path");
-    const table = db.createTable(
-        "table",
-        [{ a: 1, b: 2}],
-        {storageOptions: {timeout: "60s"}}
-    );
-    ```
-
-!!! info "Storage option casing"
-
-    The storage option keys are case-insensitive. So `connect_timeout` and `CONNECT_TIMEOUT` are the same setting. Usually lowercase is used in the `storage_options` argument and uppercase is used for environment variables. In the `lancedb` Node package, the keys can also be provided in `camelCase` capitalization. For example, `connectTimeout` is equivalent to `connect_timeout`.
-
-### General configuration
-
-There are several options that can be set for all object stores, mostly related to network client configuration.
-
-<!-- from here: https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html -->
-
-| Key                        | Description                                                                                      |
-|----------------------------|--------------------------------------------------------------------------------------------------|
-| `allow_http`               | Allow non-TLS, i.e. non-HTTPS connections. Default: `False`.                                      |
-| `allow_invalid_certificates`| Skip certificate validation on HTTPS connections. Default: `False`.                               |
-| `connect_timeout`          | Timeout for only the connect phase of a Client. Default: `5s`.                                    |
-| `timeout`                  | Timeout for the entire request, from connection until the response body has finished. Default: `30s`. |
-| `user_agent`               | User agent string to use in requests.                                                             |
-| `proxy_url`                | URL of a proxy server to use for requests. Default: `None`.                                       |
-| `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
-| `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
+LanceDB OSS uses the [object-store](https://docs.rs/object_store/latest/object_store/) Rust crate for object store access. There are general environment variables that can be used to configure the object store, such as the request timeout and proxy configuration. See the [object_store ClientConfigKey](https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html) doc for available configuration options. The environment variables that can be set are the snake-cased versions of these variable names. For example, to set `ProxyUrl` use the environment variable `PROXY_URL`. (Don't let the Rust docs intimidate you! We link to them so you can see an up-to-date list of the available options.)
 
 
 ### AWS S3
 
-To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
-These can be set as environment variables or passed in the `storage_options` parameter:
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "aws_access_key_id": "my-access-key",
-            "aws_secret_access_key": "my-secret-key",
-            "aws_session_token": "my-session-token",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://bucket/path",
-        {
-            storageOptions: {
-                awsAccessKeyId: "my-access-key",
-                awsSecretAccessKey: "my-secret-key",
-                awsSessionToken: "my-session-token",
-            }
-        }
-    );
-    ```
+To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables.
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
-
-| Key                                | Description                                                                                          |
-|------------------------------------|------------------------------------------------------------------------------------------------------|
-| `aws_region` / `region`             | The AWS region the bucket is in. This can be automatically detected when using AWS S3, but must be specified for S3-compatible stores. |
-| `aws_access_key_id` / `access_key_id` | The AWS access key ID to use.                                                                       |
-| `aws_secret_access_key` / `secret_access_key` | The AWS secret access key to use.                                                               |
-| `aws_session_token` / `session_token` | The AWS session token to use.                                                                     |
-| `aws_endpoint` / `endpoint`         | The endpoint to use for S3-compatible stores.                                                       |
-| `aws_virtual_hosted_style_request` / `virtual_hosted_style_request` | Whether to use virtual hosted-style requests, where the bucket name is part of the endpoint. Meant to be used with `aws_endpoint`. Default: `False`. |
-| `aws_s3_express` / `s3_express`     | Whether to use S3 Express One Zone endpoints. Default: `False`. See more details below.             |
-| `aws_server_side_encryption`        | The server-side encryption algorithm to use. Must be one of `"AES256"`, `"aws:kms"`, or `"aws:kms:dsse"`. Default: `None`. |
-| `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
-| `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
-
+You can see a full list of environment variables [here](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html#method.from_env).
 
 !!! tip "Automatic cleanup for failed writes"
 
@@ -267,174 +146,22 @@ For **read-only access**, LanceDB will need a policy such as:
 
 #### S3-compatible stores
 
-LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "region": "us-east-1",
-            "endpoint": "http://minio:9000",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://bucket/path",
-        {
-            storageOptions: {
-                region: "us-east-1",
-                endpoint: "http://minio:9000",
-            }
-        }
-    );
-    ```
-
-This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
-
-#### S3 Express
-
-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
-
-To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://my-bucket--use1-az4--x-s3/path",
-        storage_options={
-            "region": "us-east-1",
-            "s3_express": "true",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://my-bucket--use1-az4--x-s3/path",
-        {
-            storageOptions: {
-                region: "us-east-1",
-                s3Express: "true",
-            }
-        }
-    );
-    ```
+LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify two environment variables: `AWS_ENDPOINT` and `AWS_DEFAULT_REGION`. `AWS_ENDPOINT` should be the URL of the S3-compatible store, and `AWS_DEFAULT_REGION` should be the region to use.
 
+<!-- TODO: we should also document the use of S3 Express once we fully support it -->
 
 ### Google Cloud Storage
 
-GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. Alternatively, you can pass the path to the JSON file in the `storage_options`:
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "gs://my-bucket/my-database",
-        storage_options={
-            "service_account": "path/to/service-account.json",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "gs://my-bucket/my-database",
-        {
-            storageOptions: {
-                serviceAccount: "path/to/service-account.json",
-            }
-        }
-    );
-    ```
+GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. There are several aliases for this environment variable, documented [here](https://docs.rs/object_store/latest/object_store/gcp/struct.GoogleCloudStorageBuilder.html#method.from_env).
 
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
-
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
-<!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
-
-| Key                                   | Description                                  |
-|---------------------------------------|----------------------------------------------|
-| ``google_service_account`` / `service_account` | Path to the service account JSON file.       |
-| ``google_service_account_key``        | The serialized service account key.          |
-| ``google_application_credentials``    | Path to the application credentials.         |
-
-
 ### Azure Blob Storage
 
-Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
+Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME` and ``AZURE_STORAGE_ACCOUNT_KEY`` environment variables. The full list of environment variables that can be set are documented [here](https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html#method.from_env).
 
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "az://my-container/my-database",
-        storage_options={
-            account_name: "some-account",
-            account_key: "some-key",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "az://my-container/my-database",
-        {
-            storageOptions: {
-                accountName: "some-account",
-                accountKey: "some-key",
-            }
-        }
-    );
-    ```
-
-These keys can be used as both environment variables or keys in the `storage_options` parameter:
-
-<!-- source: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html -->
-
-| Key                                   | Description                                                                                      |
-|---------------------------------------|--------------------------------------------------------------------------------------------------|
-| ``azure_storage_account_name``        | The name of the azure storage account.                                                           |
-| ``azure_storage_account_key``         | The serialized service account key.                                                              |
-| ``azure_client_id``                   | Service principal client id for authorizing requests.                                            |
-| ``azure_client_secret``               | Service principal client secret for authorizing requests.                                        |
-| ``azure_tenant_id``                   | Tenant id used in oauth flows.                                                                   |
-| ``azure_storage_sas_key``             | Shared access signature. The signature is expected to be percent-encoded, much like they are provided in the azure storage explorer or azure portal. |
-| ``azure_storage_token``               | Bearer token.                                                                                    |
-| ``azure_storage_use_emulator``        | Use object store with azurite storage emulator.                                                  |
-| ``azure_endpoint``                    | Override the endpoint used to communicate with blob storage.                                      |
-| ``azure_use_fabric_endpoint``         | Use object store with url scheme account.dfs.fabric.microsoft.com.                               |
-| ``azure_msi_endpoint``                | Endpoint to request a imds managed identity token.                                               |
-| ``azure_object_id``                   | Object id for use with managed identity authentication.                                          |
-| ``azure_msi_resource_id``             | Msi resource id for use with managed identity authentication.                                    |
-| ``azure_federated_token_file``        | File containing token for Azure AD workload identity federation.                                 |
-| ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
-| ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
 
 <!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables.md

@@ -168,24 +168,24 @@ This guide will show how to create tables, insert data into them, and update the
     --8<-- "docs/src/basic_legacy.ts:create_f16_table"
     ```
 
-### From Pydantic Models
+    ### From Pydantic Models
 
-When you create an empty table without data, you must specify the table schema.
-LanceDB supports creating tables by specifying a PyArrow schema or a specialized
-Pydantic model called `LanceModel`.
+    When you create an empty table without data, you must specify the table schema.
+    LanceDB supports creating tables by specifying a PyArrow schema or a specialized
+    Pydantic model called `LanceModel`.
 
-For example, the following Content model specifies a table with 5 columns:
-`movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
-pass the class as the value of the `schema` parameter to `create_table`.
-The `vector` column is a `Vector` type, which is a specialized Pydantic type that
-can be configured with the vector dimensions. It is also important to note that
-LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
-(which itself derives from `pydantic.BaseModel`).
+    For example, the following Content model specifies a table with 5 columns:
+    `movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
+    pass the class as the value of the `schema` parameter to `create_table`.
+    The `vector` column is a `Vector` type, which is a specialized Pydantic type that
+    can be configured with the vector dimensions. It is also important to note that
+    LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
+    (which itself derives from `pydantic.BaseModel`).
 
-```python
-from lancedb.pydantic import Vector, LanceModel
+    ```python
+    from lancedb.pydantic import Vector, LanceModel
 
-class Content(LanceModel):
+    class Content(LanceModel):
         movie_id: int
         vector: Vector(128)
         genres: str
@@ -196,65 +196,65 @@ class Content(LanceModel):
         def imdb_url(self) -> str:
             return f"https://www.imdb.com/title/tt{self.imdb_id}"
 
-import pyarrow as pa
-db = lancedb.connect("~/.lancedb")
-table_name = "movielens_small"
-table = db.create_table(table_name, schema=Content)
-```
+    import pyarrow as pa
+    db = lancedb.connect("~/.lancedb")
+    table_name = "movielens_small"
+    table = db.create_table(table_name, schema=Content)
+    ```
 
-#### Nested schemas
+    #### Nested schemas
 
-Sometimes your data model may contain nested objects.
-For example, you may want to store the document string
-and the document soure name as a nested Document object:
+    Sometimes your data model may contain nested objects.
+    For example, you may want to store the document string
+    and the document soure name as a nested Document object:
 
-```python
-class Document(BaseModel):
+    ```python
+    class Document(BaseModel):
         content: str
         source: str
-```
+    ```
 
-This can be used as the type of a LanceDB table column:
+    This can be used as the type of a LanceDB table column:
 
-```python
-class NestedSchema(LanceModel):
+    ```python
+    class NestedSchema(LanceModel):
         id: str
         vector: Vector(1536)
         document: Document
 
-tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
-```
+    tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
+    ```
 
-This creates a struct column called "document" that has two subfields 
-called "content" and "source":
+    This creates a struct column called "document" that has two subfields 
+    called "content" and "source":
 
-```
-In [28]: tbl.schema
-Out[28]:
-id: string not null
-vector: fixed_size_list<item: float>[1536] not null
+    ```
+    In [28]: tbl.schema
+    Out[28]:
+    id: string not null
+    vector: fixed_size_list<item: float>[1536] not null
         child 0, item: float
-document: struct<content: string not null, source: string not null> not null
+    document: struct<content: string not null, source: string not null> not null
         child 0, content: string not null
         child 1, source: string not null    
-```
+    ```
 
-#### Validators
+    #### Validators
 
-Note that neither Pydantic nor PyArrow automatically validates that input data
-is of the correct timezone, but this is easy to add as a custom field validator:
+    Note that neither Pydantic nor PyArrow automatically validates that input data
+    is of the correct timezone, but this is easy to add as a custom field validator:
 
-```python
-from datetime import datetime
-from zoneinfo import ZoneInfo
+    ```python
+    from datetime import datetime
+    from zoneinfo import ZoneInfo
 
-from lancedb.pydantic import LanceModel
-from pydantic import Field, field_validator, ValidationError, ValidationInfo
+    from lancedb.pydantic import LanceModel
+    from pydantic import Field, field_validator, ValidationError, ValidationInfo
 
-tzname = "America/New_York"
-tz = ZoneInfo(tzname)
+    tzname = "America/New_York"
+    tz = ZoneInfo(tzname)
 
-class TestModel(LanceModel):
+    class TestModel(LanceModel):
         dt_with_tz: datetime = Field(json_schema_extra={"tz": tzname})
 
         @field_validator('dt_with_tz')
@@ -263,35 +263,35 @@ class TestModel(LanceModel):
             assert dt.tzinfo == tz
             return dt        
 
-ok = TestModel(dt_with_tz=datetime.now(tz))
+    ok = TestModel(dt_with_tz=datetime.now(tz))
 
-try:
+    try:
         TestModel(dt_with_tz=datetime.now(ZoneInfo("Asia/Shanghai")))
         assert 0 == 1, "this should raise ValidationError"
-except ValidationError:
+    except ValidationError:
         print("A ValidationError was raised.")
         pass
-```
+    ```
 
-When you run this code it should print "A ValidationError was raised."
+    When you run this code it should print "A ValidationError was raised."
 
-#### Pydantic custom types
+    #### Pydantic custom types
 
-LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
-please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
+    LanceDB does NOT yet support converting pydantic custom types. If this is something you need,
+    please file a feature request on the [LanceDB Github repo](https://github.com/lancedb/lancedb/issues/new).
 
-### Using Iterators / Writing Large Datasets
+    ### Using Iterators / Writing Large Datasets
 
-It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+    It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
 
-LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
+    LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
 
-Here's an example using using `RecordBatch` iterator for creating tables.
+    Here's an example using using `RecordBatch` iterator for creating tables.
 
-```python
-import pyarrow as pa
+    ```python
+    import pyarrow as pa
 
-def make_batches():
+    def make_batches():
         for i in range(5):
             yield pa.RecordBatch.from_arrays(
                 [
@@ -303,16 +303,16 @@ def make_batches():
                 ["vector", "item", "price"],
             )
 
-schema = pa.schema([
+    schema = pa.schema([
         pa.field("vector", pa.list_(pa.float32(), 4)),
         pa.field("item", pa.utf8()),
         pa.field("price", pa.float32()),
-])
+    ])
 
-db.create_table("batched_tale", make_batches(), schema=schema)
-```
+    db.create_table("batched_tale", make_batches(), schema=schema)
+    ```
 
-You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
+    You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
 ## Open existing tables
 

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverles
 
 * Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-* Python, Javascript/Typescript, and Rust support
+* Native Python and Javascript/Typescript support
 
 * Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
@@ -54,4 +54,3 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
 * [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references
-* [Rust API Reference](https://docs.rs/lancedb/latest/lancedb/index.html): Rust API reference

docs/src/integrations/index.md

@@ -13,7 +13,7 @@ Get started using these examples and quick links.
 | Integrations | |
 |---|---:|
 | <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/docs/integrations/vectorstores/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
 | <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/js/.nojekyll

@@ -1 +0,0 @@
-TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/js/README.md

@@ -1,83 +0,0 @@
-@lancedb/lancedb / [Exports](modules.md)
-
-# LanceDB JavaScript SDK
-
-A JavaScript library for [LanceDB](https://github.com/lancedb/lancedb).
-
-## Installation
-
-```bash
-npm install @lancedb/lancedb
-```
-
-This will download the appropriate native library for your platform. We currently
-support:
-
-- Linux (x86_64 and aarch64)
-- MacOS (Intel and ARM/M1/M2)
-- Windows (x86_64 only)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
-
-## Usage
-
-### Basic Example
-
-```javascript
-import * as lancedb from "@lancedb/lancedb";
-const db = await lancedb.connect("data/sample-lancedb");
-const table = await db.createTable("my_table", [
-  { id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
-  { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 },
-]);
-const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
-console.log(results);
-```
-
-The [quickstart](../basic.md) contains a more complete example.
-
-## Development
-
-```sh
-npm run build
-npm run test
-```
-
-### Running lint / format
-
-LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
-may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
-set to true. Also, if your vscode root folder is the repo root then you will need to set
-the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
-
-```sh
-npm run lint
-```
-
-LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
-"Prettier - Code formatter" extension. You should then configure it to be the default formatter
-for typescript and you should enable format on save. To manually check your code's format you
-can run:
-
-```sh
-npm run chkformat
-```
-
-If you need to manually format your code you can run:
-
-```sh
-npx prettier --write .
-```
-
-### Generating docs
-
-```sh
-npm run docs
-
-cd ../docs
-# Asssume the virtual environment was created
-# python3 -m venv venv
-# pip install -r requirements.txt
-. ./venv/bin/activate
-mkdocs build
-```

docs/src/js/classes/Connection.md

@@ -1,239 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Connection
-
-# Class: Connection
-
-A LanceDB Connection that allows you to open tables and create new ones.
-
-Connection could be local against filesystem or remote against a server.
-
-A Connection is intended to be a long lived object and may hold open
-resources such as HTTP connection pools.  This is generally fine and
-a single connection should be shared if it is going to be used many
-times. However, if you are finished with a connection, you may call
-close to eagerly free these resources.  Any call to a Connection
-method after it has been closed will result in an error.
-
-Closing a connection is optional.  Connections will automatically
-be closed when they are garbage collected.
-
-Any created tables are independent and will continue to work even if
-the underlying connection has been closed.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Connection.md#constructor)
-
-### Properties
-
-- [inner](Connection.md#inner)
-
-### Methods
-
-- [close](Connection.md#close)
-- [createEmptyTable](Connection.md#createemptytable)
-- [createTable](Connection.md#createtable)
-- [display](Connection.md#display)
-- [dropTable](Connection.md#droptable)
-- [isOpen](Connection.md#isopen)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new Connection**(`inner`): [`Connection`](Connection.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Connection` |
-
-#### Returns
-
-[`Connection`](Connection.md)
-
-#### Defined in
-
-[connection.ts:72](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L72)
-
-## Properties
-
-### inner
-
-• `Readonly` **inner**: `Connection`
-
-#### Defined in
-
-[connection.ts:70](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L70)
-
-## Methods
-
-### close
-
-▸ **close**(): `void`
-
-Close the connection, releasing any underlying resources.
-
-It is safe to call this method multiple times.
-
-Any attempt to use the connection after it is closed will result in an error.
-
-#### Returns
-
-`void`
-
-#### Defined in
-
-[connection.ts:88](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L88)
-
-___
-
-### createEmptyTable
-
-▸ **createEmptyTable**(`name`, `schema`, `options?`): `Promise`\<[`Table`](Table.md)\>
-
-Creates a new empty Table
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `schema` | `Schema`\<`any`\> | The schema of the table |
-| `options?` | `Partial`\<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)\> | - |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\>
-
-#### Defined in
-
-[connection.ts:151](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L151)
-
-___
-
-### createTable
-
-▸ **createTable**(`name`, `data`, `options?`): `Promise`\<[`Table`](Table.md)\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `options?` | `Partial`\<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)\> | - |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\>
-
-#### Defined in
-
-[connection.ts:123](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L123)
-
-___
-
-### display
-
-▸ **display**(): `string`
-
-Return a brief description of the connection
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[connection.ts:93](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L93)
-
-___
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`\<`void`\>
-
-Drop an existing table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table to drop. |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[connection.ts:173](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L173)
-
-___
-
-### isOpen
-
-▸ **isOpen**(): `boolean`
-
-Return true if the connection has not been closed
-
-#### Returns
-
-`boolean`
-
-#### Defined in
-
-[connection.ts:77](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L77)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`\<[`Table`](Table.md)\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table |
-
-#### Returns
-
-`Promise`\<[`Table`](Table.md)\>
-
-#### Defined in
-
-[connection.ts:112](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L112)
-
-___
-
-### tableNames
-
-▸ **tableNames**(`options?`): `Promise`\<`string`[]\>
-
-List all the table names in this database.
-
-Tables will be returned in lexicographical order.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `options?` | `Partial`\<[`TableNamesOptions`](../interfaces/TableNamesOptions.md)\> | options to control the paging / start point |
-
-#### Returns
-
-`Promise`\<`string`[]\>
-
-#### Defined in
-
-[connection.ts:104](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L104)

docs/src/js/classes/Index.md

@@ -1,121 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Index
-
-# Class: Index
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Index.md#constructor)
-
-### Properties
-
-- [inner](Index.md#inner)
-
-### Methods
-
-- [btree](Index.md#btree)
-- [ivfPq](Index.md#ivfpq)
-
-## Constructors
-
-### constructor
-
-• **new Index**(`inner`): [`Index`](Index.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Index` |
-
-#### Returns
-
-[`Index`](Index.md)
-
-#### Defined in
-
-[indices.ts:118](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L118)
-
-## Properties
-
-### inner
-
-• `Private` `Readonly` **inner**: `Index`
-
-#### Defined in
-
-[indices.ts:117](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L117)
-
-## Methods
-
-### btree
-
-▸ **btree**(): [`Index`](Index.md)
-
-Create a btree index
-
-A btree index is an index on a scalar columns.  The index stores a copy of the column
-in sorted order.  A header entry is created for each block of rows (currently the
-block size is fixed at 4096).  These header entries are stored in a separate
-cacheable structure (a btree).  To search for data the header is used to determine
-which blocks need to be read from disk.
-
-For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
-bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
-the correct row ids.
-
-This index is good for scalar columns with mostly distinct values and does best when
-the query is highly selective.
-
-The btree index does not currently have any parameters though parameters such as the
-block size may be added in the future.
-
-#### Returns
-
-[`Index`](Index.md)
-
-#### Defined in
-
-[indices.ts:175](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L175)
-
-___
-
-### ivfPq
-
-▸ **ivfPq**(`options?`): [`Index`](Index.md)
-
-Create an IvfPq index
-
-This index stores a compressed (quantized) copy of every vector.  These vectors
-are grouped into partitions of similar vectors.  Each partition keeps track of
-a centroid which is the average value of all vectors in the group.
-
-During a query the centroids are compared with the query vector to find the closest
-partitions.  The compressed vectors in these partitions are then searched to find
-the closest vectors.
-
-The compression scheme is called product quantization.  Each vector is divided into
-subvectors and then each subvector is quantized into a small number of bits.  the
-parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
-between index size (and thus search speed) and index accuracy.
-
-The partitioning process is called IVF and the `num_partitions` parameter controls how
-many groups to create.
-
-Note that training an IVF PQ index on a large dataset is a slow operation and
-currently is also a memory intensive operation.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `options?` | `Partial`\<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)\> |
-
-#### Returns
-
-[`Index`](Index.md)
-
-#### Defined in
-
-[indices.ts:144](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L144)

docs/src/js/classes/MakeArrowTableOptions.md

@@ -1,75 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / MakeArrowTableOptions
-
-# Class: MakeArrowTableOptions
-
-Options to control the makeArrowTable call.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](MakeArrowTableOptions.md#constructor)
-
-### Properties
-
-- [dictionaryEncodeStrings](MakeArrowTableOptions.md#dictionaryencodestrings)
-- [schema](MakeArrowTableOptions.md#schema)
-- [vectorColumns](MakeArrowTableOptions.md#vectorcolumns)
-
-## Constructors
-
-### constructor
-
-• **new MakeArrowTableOptions**(`values?`): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `values?` | `Partial`\<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)\> |
-
-#### Returns
-
-[`MakeArrowTableOptions`](MakeArrowTableOptions.md)
-
-#### Defined in
-
-[arrow.ts:100](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L100)
-
-## Properties
-
-### dictionaryEncodeStrings
-
-• **dictionaryEncodeStrings**: `boolean` = `false`
-
-If true then string columns will be encoded with dictionary encoding
-
-Set this to true if your string columns tend to repeat the same values
-often.  For more precise control use the `schema` property to specify the
-data type for individual columns.
-
-If `schema` is provided then this property is ignored.
-
-#### Defined in
-
-[arrow.ts:98](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L98)
-
-___
-
-### schema
-
-• `Optional` **schema**: `Schema`\<`any`\>
-
-#### Defined in
-
-[arrow.ts:67](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L67)
-
-___
-
-### vectorColumns
-
-• **vectorColumns**: `Record`\<`string`, [`VectorColumnOptions`](VectorColumnOptions.md)\>
-
-#### Defined in
-
-[arrow.ts:85](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L85)

docs/src/js/classes/Query.md

@@ -1,368 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Query
-
-# Class: Query
-
-A builder for LanceDB queries.
-
-## Hierarchy
-
-- [`QueryBase`](QueryBase.md)\<`NativeQuery`, [`Query`](Query.md)\>
-
-  ↳ **`Query`**
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Query.md#constructor)
-
-### Properties
-
-- [inner](Query.md#inner)
-
-### Methods
-
-- [[asyncIterator]](Query.md#[asynciterator])
-- [execute](Query.md#execute)
-- [limit](Query.md#limit)
-- [nativeExecute](Query.md#nativeexecute)
-- [nearestTo](Query.md#nearestto)
-- [select](Query.md#select)
-- [toArray](Query.md#toarray)
-- [toArrow](Query.md#toarrow)
-- [where](Query.md#where)
-
-## Constructors
-
-### constructor
-
-• **new Query**(`tbl`): [`Query`](Query.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `Table` |
-
-#### Returns
-
-[`Query`](Query.md)
-
-#### Overrides
-
-[QueryBase](QueryBase.md).[constructor](QueryBase.md#constructor)
-
-#### Defined in
-
-[query.ts:329](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L329)
-
-## Properties
-
-### inner
-
-• `Protected` **inner**: `Query`
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[inner](QueryBase.md#inner)
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
-
-## Methods
-
-### [asyncIterator]
-
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Returns
-
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[[asyncIterator]](QueryBase.md#[asynciterator])
-
-#### Defined in
-
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
-
-___
-
-### execute
-
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
-
-Execute the query and return the results as an
-
-#### Returns
-
-[`RecordBatchIterator`](RecordBatchIterator.md)
-
-**`See`**
-
- - AsyncIterator
-of
- - RecordBatch.
-
-By default, LanceDb will use many threads to calculate results and, when
-the result set is large, multiple batches will be processed at one time.
-This readahead is limited however and backpressure will be applied if this
-stream is consumed slowly (this constrains the maximum memory used by a
-single query)
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[execute](QueryBase.md#execute)
-
-#### Defined in
-
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
-
-___
-
-### limit
-
-▸ **limit**(`limit`): [`Query`](Query.md)
-
-Set the maximum number of results to return.
-
-By default, a plain search has no limit.  If this method is not
-called then every valid row from the table will be returned.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `limit` | `number` |
-
-#### Returns
-
-[`Query`](Query.md)
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[limit](QueryBase.md#limit)
-
-#### Defined in
-
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
-
-___
-
-### nativeExecute
-
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
-
-#### Returns
-
-`Promise`\<`RecordBatchIterator`\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[nativeExecute](QueryBase.md#nativeexecute)
-
-#### Defined in
-
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
-
-___
-
-### nearestTo
-
-▸ **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)
-
-Find the nearest vectors to the given query vector.
-
-This converts the query from a plain query to a vector query.
-
-This method will attempt to convert the input to the query vector
-expected by the embedding model.  If the input cannot be converted
-then an error will be thrown.
-
-By default, there is no embedding model, and the input should be
-an array-like object of numbers (something that can be used as input
-to Float32Array.from)
-
-If there is only one vector column (a column whose data type is a
-fixed size list of floats) then the column does not need to be specified.
-If there is more than one vector column you must use
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `vector` | `unknown` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`See`**
-
- - [VectorQuery#column](VectorQuery.md#column)  to specify which column you would like
-to compare with.
-
-If no index has been created on the vector column then a vector query
-will perform a distance comparison between the query vector and every
-vector in the database and then sort the results.  This is sometimes
-called a "flat search"
-
-For small databases, with a few hundred thousand vectors or less, this can
-be reasonably fast.  In larger databases you should create a vector index
-on the column.  If there is a vector index then an "approximate" nearest
-neighbor search (frequently called an ANN search) will be performed.  This
-search is much faster, but the results will be approximate.
-
-The query can be further parameterized using the returned builder.  There
-are various ANN search parameters that will let you fine tune your recall
-accuracy vs search latency.
-
-Vector searches always have a `limit`.  If `limit` has not been called then
-a default `limit` of 10 will be used.
- - [Query#limit](Query.md#limit)
-
-#### Defined in
-
-[query.ts:370](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L370)
-
-___
-
-### select
-
-▸ **select**(`columns`): [`Query`](Query.md)
-
-Return only the specified columns.
-
-By default a query will return all columns from the table.  However, this can have
-a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
-means we can finely tune our I/O to select exactly the columns we need.
-
-As a best practice you should always limit queries to the columns that you need.  If you
-pass in an array of column names then only those columns will be returned.
-
-You can also use this method to create new "dynamic" columns based on your existing columns.
-For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
-seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
-
-To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
-for each entry in the map.  The key provides the name of the column.  The value is
-an SQL string used to specify how the column is calculated.
-
-For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
-input to this method would be:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
-
-#### Returns
-
-[`Query`](Query.md)
-
-**`Example`**
-
-```ts
-new Map([["combined", "a + b"], ["c", "c"]])
-
-Columns will always be returned in the order given, even if that order is different than
-the order used when adding the data.
-
-Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
-uses `Object.entries` which should preserve the insertion order of the object.  However,
-object insertion order is easy to get wrong and `Map` is more foolproof.
-```
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[select](QueryBase.md#select)
-
-#### Defined in
-
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
-
-___
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
-
-Collect the results as an array of objects.
-
-#### Returns
-
-`Promise`\<`unknown`[]\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[toArray](QueryBase.md#toarray)
-
-#### Defined in
-
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
-
-___
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
-
-Collect the results as an Arrow
-
-#### Returns
-
-`Promise`\<`Table`\<`any`\>\>
-
-**`See`**
-
-ArrowTable.
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[toArrow](QueryBase.md#toarrow)
-
-#### Defined in
-
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
-
-___
-
-### where
-
-▸ **where**(`predicate`): [`Query`](Query.md)
-
-A filter statement to be applied to this query.
-
-The filter should be supplied as an SQL query string.  For example:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `predicate` | `string` |
-
-#### Returns
-
-[`Query`](Query.md)
-
-**`Example`**
-
-```ts
-x > 10
-y > 0 AND y < 100
-x > 5 OR y = 'test'
-
-Filtering performance can often be improved by creating a scalar index
-on the filter column(s).
-```
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[where](QueryBase.md#where)
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/QueryBase.md

@@ -1,291 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / QueryBase
-
-# Class: QueryBase\<NativeQueryType, QueryType\>
-
-Common methods supported by all query types
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `NativeQueryType` | extends `NativeQuery` \| `NativeVectorQuery` |
-| `QueryType` | `QueryType` |
-
-## Hierarchy
-
-- **`QueryBase`**
-
-  ↳ [`Query`](Query.md)
-
-  ↳ [`VectorQuery`](VectorQuery.md)
-
-## Implements
-
-- `AsyncIterable`\<`RecordBatch`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](QueryBase.md#constructor)
-
-### Properties
-
-- [inner](QueryBase.md#inner)
-
-### Methods
-
-- [[asyncIterator]](QueryBase.md#[asynciterator])
-- [execute](QueryBase.md#execute)
-- [limit](QueryBase.md#limit)
-- [nativeExecute](QueryBase.md#nativeexecute)
-- [select](QueryBase.md#select)
-- [toArray](QueryBase.md#toarray)
-- [toArrow](QueryBase.md#toarrow)
-- [where](QueryBase.md#where)
-
-## Constructors
-
-### constructor
-
-• **new QueryBase**\<`NativeQueryType`, `QueryType`\>(`inner`): [`QueryBase`](QueryBase.md)\<`NativeQueryType`, `QueryType`\>
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `NativeQueryType` | extends `Query` \| `VectorQuery` |
-| `QueryType` | `QueryType` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `NativeQueryType` |
-
-#### Returns
-
-[`QueryBase`](QueryBase.md)\<`NativeQueryType`, `QueryType`\>
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
-
-## Properties
-
-### inner
-
-• `Protected` **inner**: `NativeQueryType`
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
-
-## Methods
-
-### [asyncIterator]
-
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Returns
-
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Implementation of
-
-AsyncIterable.[asyncIterator]
-
-#### Defined in
-
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
-
-___
-
-### execute
-
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
-
-Execute the query and return the results as an
-
-#### Returns
-
-[`RecordBatchIterator`](RecordBatchIterator.md)
-
-**`See`**
-
- - AsyncIterator
-of
- - RecordBatch.
-
-By default, LanceDb will use many threads to calculate results and, when
-the result set is large, multiple batches will be processed at one time.
-This readahead is limited however and backpressure will be applied if this
-stream is consumed slowly (this constrains the maximum memory used by a
-single query)
-
-#### Defined in
-
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
-
-___
-
-### limit
-
-▸ **limit**(`limit`): `QueryType`
-
-Set the maximum number of results to return.
-
-By default, a plain search has no limit.  If this method is not
-called then every valid row from the table will be returned.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `limit` | `number` |
-
-#### Returns
-
-`QueryType`
-
-#### Defined in
-
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
-
-___
-
-### nativeExecute
-
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
-
-#### Returns
-
-`Promise`\<`RecordBatchIterator`\>
-
-#### Defined in
-
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
-
-___
-
-### select
-
-▸ **select**(`columns`): `QueryType`
-
-Return only the specified columns.
-
-By default a query will return all columns from the table.  However, this can have
-a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
-means we can finely tune our I/O to select exactly the columns we need.
-
-As a best practice you should always limit queries to the columns that you need.  If you
-pass in an array of column names then only those columns will be returned.
-
-You can also use this method to create new "dynamic" columns based on your existing columns.
-For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
-seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
-
-To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
-for each entry in the map.  The key provides the name of the column.  The value is
-an SQL string used to specify how the column is calculated.
-
-For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
-input to this method would be:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
-
-#### Returns
-
-`QueryType`
-
-**`Example`**
-
-```ts
-new Map([["combined", "a + b"], ["c", "c"]])
-
-Columns will always be returned in the order given, even if that order is different than
-the order used when adding the data.
-
-Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
-uses `Object.entries` which should preserve the insertion order of the object.  However,
-object insertion order is easy to get wrong and `Map` is more foolproof.
-```
-
-#### Defined in
-
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
-
-___
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
-
-Collect the results as an array of objects.
-
-#### Returns
-
-`Promise`\<`unknown`[]\>
-
-#### Defined in
-
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
-
-___
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
-
-Collect the results as an Arrow
-
-#### Returns
-
-`Promise`\<`Table`\<`any`\>\>
-
-**`See`**
-
-ArrowTable.
-
-#### Defined in
-
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
-
-___
-
-### where
-
-▸ **where**(`predicate`): `QueryType`
-
-A filter statement to be applied to this query.
-
-The filter should be supplied as an SQL query string.  For example:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `predicate` | `string` |
-
-#### Returns
-
-`QueryType`
-
-**`Example`**
-
-```ts
-x > 10
-y > 0 AND y < 100
-x > 5 OR y = 'test'
-
-Filtering performance can often be improved by creating a scalar index
-on the filter column(s).
-```
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/RecordBatchIterator.md

@@ -1,80 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / RecordBatchIterator
-
-# Class: RecordBatchIterator
-
-## Implements
-
-- `AsyncIterator`\<`RecordBatch`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](RecordBatchIterator.md#constructor)
-
-### Properties
-
-- [inner](RecordBatchIterator.md#inner)
-- [promisedInner](RecordBatchIterator.md#promisedinner)
-
-### Methods
-
-- [next](RecordBatchIterator.md#next)
-
-## Constructors
-
-### constructor
-
-• **new RecordBatchIterator**(`promise?`): [`RecordBatchIterator`](RecordBatchIterator.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `promise?` | `Promise`\<`RecordBatchIterator`\> |
-
-#### Returns
-
-[`RecordBatchIterator`](RecordBatchIterator.md)
-
-#### Defined in
-
-[query.ts:27](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L27)
-
-## Properties
-
-### inner
-
-• `Private` `Optional` **inner**: `RecordBatchIterator`
-
-#### Defined in
-
-[query.ts:25](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L25)
-
-___
-
-### promisedInner
-
-• `Private` `Optional` **promisedInner**: `Promise`\<`RecordBatchIterator`\>
-
-#### Defined in
-
-[query.ts:24](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L24)
-
-## Methods
-
-### next
-
-▸ **next**(): `Promise`\<`IteratorResult`\<`RecordBatch`\<`any`\>, `any`\>\>
-
-#### Returns
-
-`Promise`\<`IteratorResult`\<`RecordBatch`\<`any`\>, `any`\>\>
-
-#### Implementation of
-
-AsyncIterator.next
-
-#### Defined in
-
-[query.ts:33](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L33)

docs/src/js/classes/Table.md

@@ -1,594 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Table
-
-# Class: Table
-
-A Table is a collection of Records in a LanceDB Database.
-
-A Table object is expected to be long lived and reused for multiple operations.
-Table objects will cache a certain amount of index data in memory.  This cache
-will be freed when the Table is garbage collected.  To eagerly free the cache you
-can call the `close` method.  Once the Table is closed, it cannot be used for any
-further operations.
-
-Closing a table is optional.  It not closed, it will be closed when it is garbage
-collected.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Table.md#constructor)
-
-### Properties
-
-- [inner](Table.md#inner)
-
-### Methods
-
-- [add](Table.md#add)
-- [addColumns](Table.md#addcolumns)
-- [alterColumns](Table.md#altercolumns)
-- [checkout](Table.md#checkout)
-- [checkoutLatest](Table.md#checkoutlatest)
-- [close](Table.md#close)
-- [countRows](Table.md#countrows)
-- [createIndex](Table.md#createindex)
-- [delete](Table.md#delete)
-- [display](Table.md#display)
-- [dropColumns](Table.md#dropcolumns)
-- [isOpen](Table.md#isopen)
-- [listIndices](Table.md#listindices)
-- [query](Table.md#query)
-- [restore](Table.md#restore)
-- [schema](Table.md#schema)
-- [update](Table.md#update)
-- [vectorSearch](Table.md#vectorsearch)
-- [version](Table.md#version)
-
-## Constructors
-
-### constructor
-
-• **new Table**(`inner`): [`Table`](Table.md)
-
-Construct a Table. Internal use only.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Table` |
-
-#### Returns
-
-[`Table`](Table.md)
-
-#### Defined in
-
-[table.ts:69](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L69)
-
-## Properties
-
-### inner
-
-• `Private` `Readonly` **inner**: `Table`
-
-#### Defined in
-
-[table.ts:66](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L66)
-
-## Methods
-
-### add
-
-▸ **add**(`data`, `options?`): `Promise`\<`void`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | [`Data`](../modules.md#data) | Records to be inserted into the Table |
-| `options?` | `Partial`\<[`AddDataOptions`](../interfaces/AddDataOptions.md)\> | - |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:105](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L105)
-
-___
-
-### addColumns
-
-▸ **addColumns**(`newColumnTransforms`): `Promise`\<`void`\>
-
-Add new columns with defined values.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `newColumnTransforms` | [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[] | pairs of column names and the SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns in the table. |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:261](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L261)
-
-___
-
-### alterColumns
-
-▸ **alterColumns**(`columnAlterations`): `Promise`\<`void`\>
-
-Alter the name or nullability of columns.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `columnAlterations` | [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[] | One or more alterations to apply to columns. |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:270](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L270)
-
-___
-
-### checkout
-
-▸ **checkout**(`version`): `Promise`\<`void`\>
-
-Checks out a specific version of the Table
-
-Any read operation on the table will now access the data at the checked out version.
-As a consequence, calling this method will disable any read consistency interval
-that was previously set.
-
-This is a read-only operation that turns the table into a sort of "view"
-or "detached head".  Other table instances will not be affected.  To make the change
-permanent you can use the `[Self::restore]` method.
-
-Any operation that modifies the table will fail while the table is in a checked
-out state.
-
-To return the table to a normal state use `[Self::checkout_latest]`
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `version` | `number` |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:317](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L317)
-
-___
-
-### checkoutLatest
-
-▸ **checkoutLatest**(): `Promise`\<`void`\>
-
-Ensures the table is pointing at the latest version
-
-This can be used to manually update a table when the read_consistency_interval is None
-It can also be used to undo a `[Self::checkout]` operation
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:327](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L327)
-
-___
-
-### close
-
-▸ **close**(): `void`
-
-Close the table, releasing any underlying resources.
-
-It is safe to call this method multiple times.
-
-Any attempt to use the table after it is closed will result in an error.
-
-#### Returns
-
-`void`
-
-#### Defined in
-
-[table.ts:85](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L85)
-
-___
-
-### countRows
-
-▸ **countRows**(`filter?`): `Promise`\<`number`\>
-
-Count the total number of rows in the dataset.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `filter?` | `string` |
-
-#### Returns
-
-`Promise`\<`number`\>
-
-#### Defined in
-
-[table.ts:152](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L152)
-
-___
-
-### createIndex
-
-▸ **createIndex**(`column`, `options?`): `Promise`\<`void`\>
-
-Create an index to speed up queries.
-
-Indices can be created on vector columns or scalar columns.
-Indices on vector columns will speed up vector searches.
-Indices on scalar columns will speed up filtering (in both
-vector and non-vector searches)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `column` | `string` |
-| `options?` | `Partial`\<[`IndexOptions`](../interfaces/IndexOptions.md)\> |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-**`Example`**
-
-```ts
-// If the column has a vector (fixed size list) data type then
-// an IvfPq vector index will be created.
-const table = await conn.openTable("my_table");
-await table.createIndex(["vector"]);
-```
-
-**`Example`**
-
-```ts
-// For advanced control over vector index creation you can specify
-// the index type and options.
-const table = await conn.openTable("my_table");
-await table.createIndex(["vector"], I)
-  .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
-  .build();
-```
-
-**`Example`**
-
-```ts
-// Or create a Scalar index
-await table.createIndex("my_float_col").build();
-```
-
-#### Defined in
-
-[table.ts:184](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L184)
-
-___
-
-### delete
-
-▸ **delete**(`predicate`): `Promise`\<`void`\>
-
-Delete the rows that satisfy the predicate.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `predicate` | `string` |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:157](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L157)
-
-___
-
-### display
-
-▸ **display**(): `string`
-
-Return a brief description of the table
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[table.ts:90](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L90)
-
-___
-
-### dropColumns
-
-▸ **dropColumns**(`columnNames`): `Promise`\<`void`\>
-
-Drop one or more columns from the dataset
-
-This is a metadata-only operation and does not remove the data from the
-underlying storage. In order to remove the data, you must subsequently
-call ``compact_files`` to rewrite the data without the removed columns and
-then call ``cleanup_files`` to remove the old files.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `columnNames` | `string`[] | The names of the columns to drop. These can be nested column references (e.g. "a.b.c") or top-level column names (e.g. "a"). |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:285](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L285)
-
-___
-
-### isOpen
-
-▸ **isOpen**(): `boolean`
-
-Return true if the table has not been closed
-
-#### Returns
-
-`boolean`
-
-#### Defined in
-
-[table.ts:74](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L74)
-
-___
-
-### listIndices
-
-▸ **listIndices**(): `Promise`\<[`IndexConfig`](../interfaces/IndexConfig.md)[]\>
-
-List all indices that have been created with Self::create_index
-
-#### Returns
-
-`Promise`\<[`IndexConfig`](../interfaces/IndexConfig.md)[]\>
-
-#### Defined in
-
-[table.ts:350](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L350)
-
-___
-
-### query
-
-▸ **query**(): [`Query`](Query.md)
-
-Create a [Query](Query.md) Builder.
-
-Queries allow you to search your existing data.  By default the query will
-return all the data in the table in no particular order.  The builder
-returned by this method can be used to control the query using filtering,
-vector similarity, sorting, and more.
-
-Note: By default, all columns are returned.  For best performance, you should
-only fetch the columns you need.  See [`Query::select_with_projection`] for
-more details.
-
-When appropriate, various indices and statistics based pruning will be used to
-accelerate the query.
-
-#### Returns
-
-[`Query`](Query.md)
-
-A builder that can be used to parameterize the query
-
-**`Example`**
-
-```ts
-// SQL-style filtering
-//
-// This query will return up to 1000 rows whose value in the `id` column
-// is greater than 5.  LanceDb supports a broad set of filtering functions.
-for await (const batch of table.query()
-                         .filter("id > 1").select(["id"]).limit(20)) {
- console.log(batch);
-}
-```
-
-**`Example`**
-
-```ts
-// Vector Similarity Search
-//
-// This example will find the 10 rows whose value in the "vector" column are
-// closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
-// on the "vector" column then this will perform an ANN search.
-//
-// The `refine_factor` and `nprobes` methods are used to control the recall /
-// latency tradeoff of the search.
-for await (const batch of table.query()
-                   .nearestTo([1, 2, 3])
-                   .refineFactor(5).nprobe(10)
-                   .limit(10)) {
- console.log(batch);
-}
-```
-
-**`Example`**
-
-```ts
-// Scan the full dataset
-//
-// This query will return everything in the table in no particular order.
-for await (const batch of table.query()) {
-  console.log(batch);
-}
-```
-
-#### Defined in
-
-[table.ts:238](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L238)
-
-___
-
-### restore
-
-▸ **restore**(): `Promise`\<`void`\>
-
-Restore the table to the currently checked out version
-
-This operation will fail if checkout has not been called previously
-
-This operation will overwrite the latest version of the table with a
-previous version.  Any changes made since the checked out version will
-no longer be visible.
-
-Once the operation concludes the table will no longer be in a checked
-out state and the read_consistency_interval, if any, will apply.
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:343](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L343)
-
-___
-
-### schema
-
-▸ **schema**(): `Promise`\<`Schema`\<`any`\>\>
-
-Get the schema of the table.
-
-#### Returns
-
-`Promise`\<`Schema`\<`any`\>\>
-
-#### Defined in
-
-[table.ts:95](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L95)
-
-___
-
-### update
-
-▸ **update**(`updates`, `options?`): `Promise`\<`void`\>
-
-Update existing records in the Table
-
-An update operation can be used to adjust existing values.  Use the
-returned builder to specify which columns to update.  The new value
-can be a literal value (e.g. replacing nulls with some default value)
-or an expression applied to the old value (e.g. incrementing a value)
-
-An optional condition can be specified (e.g. "only update if the old
-value is 0")
-
-Note: if your condition is something like "some_id_column == 7" and
-you are updating many rows (with different ids) then you will get
-better performance with a single [`merge_insert`] call instead of
-repeatedly calilng this method.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `updates` | `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> | the columns to update Keys in the map should specify the name of the column to update. Values in the map provide the new value of the column. These can be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions based on the row being updated (e.g. "my_col + 1") |
-| `options?` | `Partial`\<[`UpdateOptions`](../interfaces/UpdateOptions.md)\> | additional options to control the update behavior |
-
-#### Returns
-
-`Promise`\<`void`\>
-
-#### Defined in
-
-[table.ts:137](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L137)
-
-___
-
-### vectorSearch
-
-▸ **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)
-
-Search the table with a given query vector.
-
-This is a convenience method for preparing a vector query and
-is the same thing as calling `nearestTo` on the builder returned
-by `query`.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `vector` | `unknown` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`See`**
-
-[Query#nearestTo](Query.md#nearestto) for more details.
-
-#### Defined in
-
-[table.ts:249](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L249)
-
-___
-
-### version
-
-▸ **version**(): `Promise`\<`number`\>
-
-Retrieve the version of the table
-
-LanceDb supports versioning.  Every operation that modifies the table increases
-version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
-version to view the data at that point.  In addition, you can `[Self::restore]` the
-version to replace the current table with a previous version.
-
-#### Returns
-
-`Promise`\<`number`\>
-
-#### Defined in
-
-[table.ts:297](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L297)

docs/src/js/classes/VectorColumnOptions.md

@@ -1,45 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / VectorColumnOptions
-
-# Class: VectorColumnOptions
-
-## Table of contents
-
-### Constructors
-
-- [constructor](VectorColumnOptions.md#constructor)
-
-### Properties
-
-- [type](VectorColumnOptions.md#type)
-
-## Constructors
-
-### constructor
-
-• **new VectorColumnOptions**(`values?`): [`VectorColumnOptions`](VectorColumnOptions.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `values?` | `Partial`\<[`VectorColumnOptions`](VectorColumnOptions.md)\> |
-
-#### Returns
-
-[`VectorColumnOptions`](VectorColumnOptions.md)
-
-#### Defined in
-
-[arrow.ts:49](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L49)
-
-## Properties
-
-### type
-
-• **type**: `Float`\<`Floats`\>
-
-Vector column type.
-
-#### Defined in
-
-[arrow.ts:47](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L47)

docs/src/js/classes/VectorQuery.md

@@ -1,531 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / VectorQuery
-
-# Class: VectorQuery
-
-A builder used to construct a vector search
-
-This builder can be reused to execute the query many times.
-
-## Hierarchy
-
-- [`QueryBase`](QueryBase.md)\<`NativeVectorQuery`, [`VectorQuery`](VectorQuery.md)\>
-
-  ↳ **`VectorQuery`**
-
-## Table of contents
-
-### Constructors
-
-- [constructor](VectorQuery.md#constructor)
-
-### Properties
-
-- [inner](VectorQuery.md#inner)
-
-### Methods
-
-- [[asyncIterator]](VectorQuery.md#[asynciterator])
-- [bypassVectorIndex](VectorQuery.md#bypassvectorindex)
-- [column](VectorQuery.md#column)
-- [distanceType](VectorQuery.md#distancetype)
-- [execute](VectorQuery.md#execute)
-- [limit](VectorQuery.md#limit)
-- [nativeExecute](VectorQuery.md#nativeexecute)
-- [nprobes](VectorQuery.md#nprobes)
-- [postfilter](VectorQuery.md#postfilter)
-- [refineFactor](VectorQuery.md#refinefactor)
-- [select](VectorQuery.md#select)
-- [toArray](VectorQuery.md#toarray)
-- [toArrow](VectorQuery.md#toarrow)
-- [where](VectorQuery.md#where)
-
-## Constructors
-
-### constructor
-
-• **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `VectorQuery` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-#### Overrides
-
-[QueryBase](QueryBase.md).[constructor](QueryBase.md#constructor)
-
-#### Defined in
-
-[query.ts:189](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L189)
-
-## Properties
-
-### inner
-
-• `Protected` **inner**: `VectorQuery`
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[inner](QueryBase.md#inner)
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
-
-## Methods
-
-### [asyncIterator]
-
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Returns
-
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[[asyncIterator]](QueryBase.md#[asynciterator])
-
-#### Defined in
-
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
-
-___
-
-### bypassVectorIndex
-
-▸ **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)
-
-If this is called then any vector index is skipped
-
-An exhaustive (flat) search will be performed.  The query vector will
-be compared to every vector in the table.  At high scales this can be
-expensive.  However, this is often still useful.  For example, skipping
-the vector index can give you ground truth results which you can use to
-calculate your recall to select an appropriate value for nprobes.
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-#### Defined in
-
-[query.ts:321](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L321)
-
-___
-
-### column
-
-▸ **column**(`column`): [`VectorQuery`](VectorQuery.md)
-
-Set the vector column to query
-
-This controls which column is compared to the query vector supplied in
-the call to
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `column` | `string` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`See`**
-
-[Query#nearestTo](Query.md#nearestto)
-
-This parameter must be specified if the table has more than one column
-whose data type is a fixed-size-list of floats.
-
-#### Defined in
-
-[query.ts:229](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L229)
-
-___
-
-### distanceType
-
-▸ **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)
-
-Set the distance metric to use
-
-When performing a vector search we try and find the "nearest" vectors according
-to some kind of distance metric.  This parameter controls which distance metric to
-use.  See
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `distanceType` | `string` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`See`**
-
-[IvfPqOptions.distanceType](../interfaces/IvfPqOptions.md#distancetype) for more details on the different
-distance metrics available.
-
-Note: if there is a vector index then the distance type used MUST match the distance
-type used to train the vector index.  If this is not done then the results will be
-invalid.
-
-By default "l2" is used.
-
-#### Defined in
-
-[query.ts:248](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L248)
-
-___
-
-### execute
-
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
-
-Execute the query and return the results as an
-
-#### Returns
-
-[`RecordBatchIterator`](RecordBatchIterator.md)
-
-**`See`**
-
- - AsyncIterator
-of
- - RecordBatch.
-
-By default, LanceDb will use many threads to calculate results and, when
-the result set is large, multiple batches will be processed at one time.
-This readahead is limited however and backpressure will be applied if this
-stream is consumed slowly (this constrains the maximum memory used by a
-single query)
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[execute](QueryBase.md#execute)
-
-#### Defined in
-
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
-
-___
-
-### limit
-
-▸ **limit**(`limit`): [`VectorQuery`](VectorQuery.md)
-
-Set the maximum number of results to return.
-
-By default, a plain search has no limit.  If this method is not
-called then every valid row from the table will be returned.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `limit` | `number` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[limit](QueryBase.md#limit)
-
-#### Defined in
-
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
-
-___
-
-### nativeExecute
-
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
-
-#### Returns
-
-`Promise`\<`RecordBatchIterator`\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[nativeExecute](QueryBase.md#nativeexecute)
-
-#### Defined in
-
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
-
-___
-
-### nprobes
-
-▸ **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)
-
-Set the number of partitions to search (probe)
-
-This argument is only used when the vector column has an IVF PQ index.
-If there is no index then this value is ignored.
-
-The IVF stage of IVF PQ divides the input into partitions (clusters) of
-related values.
-
-The partition whose centroids are closest to the query vector will be
-exhaustiely searched to find matches.  This parameter controls how many
-partitions should be searched.
-
-Increasing this value will increase the recall of your query but will
-also increase the latency of your query.  The default value is 20.  This
-default is good for many cases but the best value to use will depend on
-your data and the recall that you need to achieve.
-
-For best results we recommend tuning this parameter with a benchmark against
-your actual data to find the smallest possible value that will still give
-you the desired recall.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `nprobes` | `number` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-#### Defined in
-
-[query.ts:215](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L215)
-
-___
-
-### postfilter
-
-▸ **postfilter**(): [`VectorQuery`](VectorQuery.md)
-
-If this is called then filtering will happen after the vector search instead of
-before.
-
-By default filtering will be performed before the vector search.  This is how
-filtering is typically understood to work.  This prefilter step does add some
-additional latency.  Creating a scalar index on the filter column(s) can
-often improve this latency.  However, sometimes a filter is too complex or scalar
-indices cannot be applied to the column.  In these cases postfiltering can be
-used instead of prefiltering to improve latency.
-
-Post filtering applies the filter to the results of the vector search.  This means
-we only run the filter on a much smaller set of data.  However, it can cause the
-query to return fewer than `limit` results (or even no results) if none of the nearest
-results match the filter.
-
-Post filtering happens during the "refine stage" (described in more detail in
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`See`**
-
-[VectorQuery#refineFactor](VectorQuery.md#refinefactor)).  This means that setting a higher refine
-factor can often help restore some of the results lost by post filtering.
-
-#### Defined in
-
-[query.ts:307](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L307)
-
-___
-
-### refineFactor
-
-▸ **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)
-
-A multiplier to control how many additional rows are taken during the refine step
-
-This argument is only used when the vector column has an IVF PQ index.
-If there is no index then this value is ignored.
-
-An IVF PQ index stores compressed (quantized) values.  They query vector is compared
-against these values and, since they are compressed, the comparison is inaccurate.
-
-This parameter can be used to refine the results.  It can improve both improve recall
-and correct the ordering of the nearest results.
-
-To refine results LanceDb will first perform an ANN search to find the nearest
-`limit` * `refine_factor` results.  In other words, if `refine_factor` is 3 and
-`limit` is the default (10) then the first 30 results will be selected.  LanceDb
-then fetches the full, uncompressed, values for these 30 results.  The results are
-then reordered by the true distance and only the nearest 10 are kept.
-
-Note: there is a difference between calling this method with a value of 1 and never
-calling this method at all.  Calling this method with any value will have an impact
-on your search latency.  When you call this method with a `refine_factor` of 1 then
-LanceDb still needs to fetch the full, uncompressed, values so that it can potentially
-reorder the results.
-
-Note: if this method is NOT called then the distances returned in the _distance column
-will be approximate distances based on the comparison of the quantized query vector
-and the quantized result vectors.  This can be considerably different than the true
-distance between the query vector and the actual uncompressed vector.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `refineFactor` | `number` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-#### Defined in
-
-[query.ts:282](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L282)
-
-___
-
-### select
-
-▸ **select**(`columns`): [`VectorQuery`](VectorQuery.md)
-
-Return only the specified columns.
-
-By default a query will return all columns from the table.  However, this can have
-a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
-means we can finely tune our I/O to select exactly the columns we need.
-
-As a best practice you should always limit queries to the columns that you need.  If you
-pass in an array of column names then only those columns will be returned.
-
-You can also use this method to create new "dynamic" columns based on your existing columns.
-For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
-seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
-
-To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
-for each entry in the map.  The key provides the name of the column.  The value is
-an SQL string used to specify how the column is calculated.
-
-For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
-input to this method would be:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`Example`**
-
-```ts
-new Map([["combined", "a + b"], ["c", "c"]])
-
-Columns will always be returned in the order given, even if that order is different than
-the order used when adding the data.
-
-Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
-uses `Object.entries` which should preserve the insertion order of the object.  However,
-object insertion order is easy to get wrong and `Map` is more foolproof.
-```
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[select](QueryBase.md#select)
-
-#### Defined in
-
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
-
-___
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
-
-Collect the results as an array of objects.
-
-#### Returns
-
-`Promise`\<`unknown`[]\>
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[toArray](QueryBase.md#toarray)
-
-#### Defined in
-
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
-
-___
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
-
-Collect the results as an Arrow
-
-#### Returns
-
-`Promise`\<`Table`\<`any`\>\>
-
-**`See`**
-
-ArrowTable.
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[toArrow](QueryBase.md#toarrow)
-
-#### Defined in
-
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
-
-___
-
-### where
-
-▸ **where**(`predicate`): [`VectorQuery`](VectorQuery.md)
-
-A filter statement to be applied to this query.
-
-The filter should be supplied as an SQL query string.  For example:
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `predicate` | `string` |
-
-#### Returns
-
-[`VectorQuery`](VectorQuery.md)
-
-**`Example`**
-
-```ts
-x > 10
-y > 0 AND y < 100
-x > 5 OR y = 'test'
-
-Filtering performance can often be improved by creating a scalar index
-on the filter column(s).
-```
-
-#### Inherited from
-
-[QueryBase](QueryBase.md).[where](QueryBase.md#where)
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/embedding.OpenAIEmbeddingFunction.md

@@ -1,111 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / [embedding](../modules/embedding.md) / OpenAIEmbeddingFunction
-
-# Class: OpenAIEmbeddingFunction
-
-[embedding](../modules/embedding.md).OpenAIEmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Implements
-
-- [`EmbeddingFunction`](../interfaces/embedding.EmbeddingFunction.md)\<`string`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](embedding.OpenAIEmbeddingFunction.md#constructor)
-
-### Properties
-
-- [\_modelName](embedding.OpenAIEmbeddingFunction.md#_modelname)
-- [\_openai](embedding.OpenAIEmbeddingFunction.md#_openai)
-- [sourceColumn](embedding.OpenAIEmbeddingFunction.md#sourcecolumn)
-
-### Methods
-
-- [embed](embedding.OpenAIEmbeddingFunction.md#embed)
-
-## Constructors
-
-### constructor
-
-• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`): [`OpenAIEmbeddingFunction`](embedding.OpenAIEmbeddingFunction.md)
-
-#### Parameters
-
-| Name | Type | Default value |
-| :------ | :------ | :------ |
-| `sourceColumn` | `string` | `undefined` |
-| `openAIKey` | `string` | `undefined` |
-| `modelName` | `string` | `"text-embedding-ada-002"` |
-
-#### Returns
-
-[`OpenAIEmbeddingFunction`](embedding.OpenAIEmbeddingFunction.md)
-
-#### Defined in
-
-[embedding/openai.ts:22](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L22)
-
-## Properties
-
-### \_modelName
-
-• `Private` `Readonly` **\_modelName**: `string`
-
-#### Defined in
-
-[embedding/openai.ts:20](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L20)
-
-___
-
-### \_openai
-
-• `Private` `Readonly` **\_openai**: `OpenAI`
-
-#### Defined in
-
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L19)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md).[sourceColumn](../interfaces/embedding.EmbeddingFunction.md#sourcecolumn)
-
-#### Defined in
-
-[embedding/openai.ts:61](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L61)
-
-## Methods
-
-### embed
-
-▸ **embed**(`data`): `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `string`[] |
-
-#### Returns
-
-`Promise`\<`number`[][]\>
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md).[embed](../interfaces/embedding.EmbeddingFunction.md#embed)
-
-#### Defined in
-
-[embedding/openai.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L48)

docs/src/js/enums/WriteMode.md

@@ -1,43 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / WriteMode
-
-# Enumeration: WriteMode
-
-Write mode for writing a table.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Append](WriteMode.md#append)
-- [Create](WriteMode.md#create)
-- [Overwrite](WriteMode.md#overwrite)
-
-## Enumeration Members
-
-### Append
-
-• **Append** = ``"Append"``
-
-#### Defined in
-
-native.d.ts:69
-
-___
-
-### Create
-
-• **Create** = ``"Create"``
-
-#### Defined in
-
-native.d.ts:68
-
-___
-
-### Overwrite
-
-• **Overwrite** = ``"Overwrite"``
-
-#### Defined in
-
-native.d.ts:70

docs/src/js/interfaces/AddColumnsSql.md

@@ -1,37 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / AddColumnsSql
-
-# Interface: AddColumnsSql
-
-A definition of a new column to add to a table.
-
-## Table of contents
-
-### Properties
-
-- [name](AddColumnsSql.md#name)
-- [valueSql](AddColumnsSql.md#valuesql)
-
-## Properties
-
-### name
-
-• **name**: `string`
-
-The name of the new column.
-
-#### Defined in
-
-native.d.ts:43
-
-___
-
-### valueSql
-
-• **valueSql**: `string`
-
-The values to populate the new column with, as a SQL expression.
-The expression can reference other columns in the table.
-
-#### Defined in
-
-native.d.ts:48

docs/src/js/interfaces/AddDataOptions.md

@@ -1,25 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / AddDataOptions
-
-# Interface: AddDataOptions
-
-Options for adding data to a table.
-
-## Table of contents
-
-### Properties
-
-- [mode](AddDataOptions.md#mode)
-
-## Properties
-
-### mode
-
-• **mode**: ``"append"`` \| ``"overwrite"``
-
-If "append" (the default) then the new data will be added to the table
-
-If "overwrite" then the new data will replace the existing data in the table.
-
-#### Defined in
-
-[table.ts:36](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L36)

docs/src/js/interfaces/ColumnAlteration.md

@@ -1,56 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ColumnAlteration
-
-# Interface: ColumnAlteration
-
-A definition of a column alteration. The alteration changes the column at
-`path` to have the new name `name`, to be nullable if `nullable` is true,
-and to have the data type `data_type`. At least one of `rename` or `nullable`
-must be provided.
-
-## Table of contents
-
-### Properties
-
-- [nullable](ColumnAlteration.md#nullable)
-- [path](ColumnAlteration.md#path)
-- [rename](ColumnAlteration.md#rename)
-
-## Properties
-
-### nullable
-
-• `Optional` **nullable**: `boolean`
-
-Set the new nullability. Note that a nullable column cannot be made non-nullable.
-
-#### Defined in
-
-native.d.ts:38
-
-___
-
-### path
-
-• **path**: `string`
-
-The path to the column to alter. This is a dot-separated path to the column.
-If it is a top-level column then it is just the name of the column. If it is
-a nested column then it is the path to the column, e.g. "a.b.c" for a column
-`c` nested inside a column `b` nested inside a column `a`.
-
-#### Defined in
-
-native.d.ts:31
-
-___
-
-### rename
-
-• `Optional` **rename**: `string`
-
-The new name of the column. If not provided then the name will not be changed.
-This must be distinct from the names of all other columns in the table.
-
-#### Defined in
-
-native.d.ts:36

docs/src/js/interfaces/ConnectionOptions.md

@@ -1,51 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ConnectionOptions
-
-# Interface: ConnectionOptions
-
-## Table of contents
-
-### Properties
-
-- [apiKey](ConnectionOptions.md#apikey)
-- [hostOverride](ConnectionOptions.md#hostoverride)
-- [readConsistencyInterval](ConnectionOptions.md#readconsistencyinterval)
-
-## Properties
-
-### apiKey
-
-• `Optional` **apiKey**: `string`
-
-#### Defined in
-
-native.d.ts:51
-
-___
-
-### hostOverride
-
-• `Optional` **hostOverride**: `string`
-
-#### Defined in
-
-native.d.ts:52
-
-___
-
-### readConsistencyInterval
-
-• `Optional` **readConsistencyInterval**: `number`
-
-(For LanceDB OSS only): The interval, in seconds, at which to check for
-updates to the table from other processes. If None, then consistency is not
-checked. For performance reasons, this is the default. For strong
-consistency, set this to zero seconds. Then every read will check for
-updates from other processes. As a compromise, you can set this to a
-non-zero value for eventual consistency. If more than that interval
-has passed since the last check, then the table will be checked for updates.
-Note: this consistency only applies to read operations. Write operations are
-always consistent.
-
-#### Defined in
-
-native.d.ts:64

docs/src/js/interfaces/CreateTableOptions.md

@@ -1,41 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / CreateTableOptions
-
-# Interface: CreateTableOptions
-
-## Table of contents
-
-### Properties
-
-- [existOk](CreateTableOptions.md#existok)
-- [mode](CreateTableOptions.md#mode)
-
-## Properties
-
-### existOk
-
-• **existOk**: `boolean`
-
-If this is true and the table already exists and the mode is "create"
-then no error will be raised.
-
-#### Defined in
-
-[connection.ts:35](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L35)
-
-___
-
-### mode
-
-• **mode**: ``"overwrite"`` \| ``"create"``
-
-The mode to use when creating the table.
-
-If this is set to "create" and the table already exists then either
-an error will be thrown or, if existOk is true, then nothing will
-happen.  Any provided data will be ignored.
-
-If this is set to "overwrite" then any existing table will be replaced.
-
-#### Defined in
-
-[connection.ts:30](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L30)

docs/src/js/interfaces/ExecutableQuery.md

@@ -1,7 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ExecutableQuery
-
-# Interface: ExecutableQuery
-
-An interface for a query that can be executed
-
-Supported by all query types

docs/src/js/interfaces/IndexConfig.md

@@ -1,39 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IndexConfig
-
-# Interface: IndexConfig
-
-A description of an index currently configured on a column
-
-## Table of contents
-
-### Properties
-
-- [columns](IndexConfig.md#columns)
-- [indexType](IndexConfig.md#indextype)
-
-## Properties
-
-### columns
-
-• **columns**: `string`[]
-
-The columns in the index
-
-Currently this is always an array of size 1.  In the future there may
-be more columns to represent composite indices.
-
-#### Defined in
-
-native.d.ts:16
-
-___
-
-### indexType
-
-• **indexType**: `string`
-
-The type of the index
-
-#### Defined in
-
-native.d.ts:9

docs/src/js/interfaces/IndexOptions.md

@@ -1,48 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IndexOptions
-
-# Interface: IndexOptions
-
-## Table of contents
-
-### Properties
-
-- [config](IndexOptions.md#config)
-- [replace](IndexOptions.md#replace)
-
-## Properties
-
-### config
-
-• `Optional` **config**: [`Index`](../classes/Index.md)
-
-Advanced index configuration
-
-This option allows you to specify a specfic index to create and also
-allows you to pass in configuration for training the index.
-
-See the static methods on Index for details on the various index types.
-
-If this is not supplied then column data type(s) and column statistics
-will be used to determine the most useful kind of index to create.
-
-#### Defined in
-
-[indices.ts:192](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L192)
-
-___
-
-### replace
-
-• `Optional` **replace**: `boolean`
-
-Whether to replace the existing index
-
-If this is false, and another index already exists on the same columns
-and the same name, then an error will be returned.  This is true even if
-that index is out of date.
-
-The default is true
-
-#### Defined in
-
-[indices.ts:202](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L202)

docs/src/js/interfaces/IvfPqOptions.md

@@ -1,144 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IvfPqOptions
-
-# Interface: IvfPqOptions
-
-Options to create an `IVF_PQ` index
-
-## Table of contents
-
-### Properties
-
-- [distanceType](IvfPqOptions.md#distancetype)
-- [maxIterations](IvfPqOptions.md#maxiterations)
-- [numPartitions](IvfPqOptions.md#numpartitions)
-- [numSubVectors](IvfPqOptions.md#numsubvectors)
-- [sampleRate](IvfPqOptions.md#samplerate)
-
-## Properties
-
-### distanceType
-
-• `Optional` **distanceType**: ``"l2"`` \| ``"cosine"`` \| ``"dot"``
-
-Distance type to use to build the index.
-
-Default value is "l2".
-
-This is used when training the index to calculate the IVF partitions
-(vectors are grouped in partitions with similar vectors according to this
-distance type) and to calculate a subvector's code during quantization.
-
-The distance type used to train an index MUST match the distance type used
-to search the index.  Failure to do so will yield inaccurate results.
-
-The following distance types are available:
-
-"l2" - Euclidean distance. This is a very common distance metric that
-accounts for both magnitude and direction when determining the distance
-between vectors. L2 distance has a range of [0, ∞).
-
-"cosine" - Cosine distance.  Cosine distance is a distance metric
-calculated from the cosine similarity between two vectors. Cosine
-similarity is a measure of similarity between two non-zero vectors of an
-inner product space. It is defined to equal the cosine of the angle
-between them.  Unlike L2, the cosine distance is not affected by the
-magnitude of the vectors.  Cosine distance has a range of [0, 2].
-
-Note: the cosine distance is undefined when one (or both) of the vectors
-are all zeros (there is no direction).  These vectors are invalid and may
-never be returned from a vector search.
-
-"dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-L2 norm is 1), then dot distance is equivalent to the cosine distance.
-
-#### Defined in
-
-[indices.ts:83](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L83)
-
-___
-
-### maxIterations
-
-• `Optional` **maxIterations**: `number`
-
-Max iteration to train IVF kmeans.
-
-When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
-controls how many iterations of kmeans to run.
-
-Increasing this might improve the quality of the index but in most cases these extra
-iterations have diminishing returns.
-
-The default value is 50.
-
-#### Defined in
-
-[indices.ts:96](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L96)
-
-___
-
-### numPartitions
-
-• `Optional` **numPartitions**: `number`
-
-The number of IVF partitions to create.
-
-This value should generally scale with the number of rows in the dataset.
-By default the number of partitions is the square root of the number of
-rows.
-
-If this value is too large then the first part of the search (picking the
-right partition) will be slow.  If this value is too small then the second
-part of the search (searching within a partition) will be slow.
-
-#### Defined in
-
-[indices.ts:32](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L32)
-
-___
-
-### numSubVectors
-
-• `Optional` **numSubVectors**: `number`
-
-Number of sub-vectors of PQ.
-
-This value controls how much the vector is compressed during the quantization step.
-The more sub vectors there are the less the vector is compressed.  The default is
-the dimension of the vector divided by 16.  If the dimension is not evenly divisible
-by 16 we use the dimension divded by 8.
-
-The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
-us to use efficient SIMD instructions.
-
-If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
-will likely result in poor performance.
-
-#### Defined in
-
-[indices.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L48)
-
-___
-
-### sampleRate
-
-• `Optional` **sampleRate**: `number`
-
-The number of vectors, per partition, to sample when training IVF kmeans.
-
-When an IVF PQ index is trained, we need to calculate partitions.  These are groups
-of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-
-Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-random sample of the data.  This parameter controls the size of the sample.  The total
-number of vectors used to train the index is `sample_rate * num_partitions`.
-
-Increasing this value might improve the quality of the index but in most cases the
-default should be sufficient.
-
-The default value is 256.
-
-#### Defined in
-
-[indices.ts:113](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L113)

docs/src/js/interfaces/TableNamesOptions.md

@@ -1,38 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / TableNamesOptions
-
-# Interface: TableNamesOptions
-
-## Table of contents
-
-### Properties
-
-- [limit](TableNamesOptions.md#limit)
-- [startAfter](TableNamesOptions.md#startafter)
-
-## Properties
-
-### limit
-
-• `Optional` **limit**: `number`
-
-An optional limit to the number of results to return.
-
-#### Defined in
-
-[connection.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L48)
-
-___
-
-### startAfter
-
-• `Optional` **startAfter**: `string`
-
-If present, only return names that come lexicographically after the
-supplied value.
-
-This can be combined with limit to implement pagination by setting this to
-the last table name from the previous page.
-
-#### Defined in
-
-[connection.ts:46](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L46)

docs/src/js/interfaces/UpdateOptions.md

@@ -1,28 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / UpdateOptions
-
-# Interface: UpdateOptions
-
-## Table of contents
-
-### Properties
-
-- [where](UpdateOptions.md#where)
-
-## Properties
-
-### where
-
-• **where**: `string`
-
-A filter that limits the scope of the update.
-
-This should be an SQL filter expression.
-
-Only rows that satisfy the expression will be updated.
-
-For example, this could be 'my_col == 0' to replace all instances
-of 0 in a column with some other default value.
-
-#### Defined in
-
-[table.ts:50](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L50)

docs/src/js/interfaces/WriteOptions.md

@@ -1,21 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / WriteOptions
-
-# Interface: WriteOptions
-
-Write options when creating a Table.
-
-## Table of contents
-
-### Properties
-
-- [mode](WriteOptions.md#mode)
-
-## Properties
-
-### mode
-
-• `Optional` **mode**: [`WriteMode`](../enums/WriteMode.md)
-
-#### Defined in
-
-native.d.ts:74

docs/src/js/interfaces/embedding.EmbeddingFunction.md

@@ -1,129 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / [embedding](../modules/embedding.md) / EmbeddingFunction
-
-# Interface: EmbeddingFunction\<T\>
-
-[embedding](../modules/embedding.md).EmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Implemented by
-
-- [`OpenAIEmbeddingFunction`](../classes/embedding.OpenAIEmbeddingFunction.md)
-
-## Table of contents
-
-### Properties
-
-- [destColumn](embedding.EmbeddingFunction.md#destcolumn)
-- [embed](embedding.EmbeddingFunction.md#embed)
-- [embeddingDataType](embedding.EmbeddingFunction.md#embeddingdatatype)
-- [embeddingDimension](embedding.EmbeddingFunction.md#embeddingdimension)
-- [excludeSource](embedding.EmbeddingFunction.md#excludesource)
-- [sourceColumn](embedding.EmbeddingFunction.md#sourcecolumn)
-
-## Properties
-
-### destColumn
-
-• `Optional` **destColumn**: `string`
-
-The name of the column that will contain the embedding
-
-By default this is "vector"
-
-#### Defined in
-
-[embedding/embedding_function.ts:49](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L49)
-
-___
-
-### embed
-
-• **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Type declaration
-
-▸ (`data`): `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-##### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `T`[] |
-
-##### Returns
-
-`Promise`\<`number`[][]\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:62](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L62)
-
-___
-
-### embeddingDataType
-
-• `Optional` **embeddingDataType**: `Float`\<`Floats`\>
-
-The data type of the embedding
-
-The embedding function should return `number`.  This will be converted into
-an Arrow float array.  By default this will be Float32 but this property can
-be used to control the conversion.
-
-#### Defined in
-
-[embedding/embedding_function.ts:33](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L33)
-
-___
-
-### embeddingDimension
-
-• `Optional` **embeddingDimension**: `number`
-
-The dimension of the embedding
-
-This is optional, normally this can be determined by looking at the results of
-`embed`.  If this is not specified, and there is an attempt to apply the embedding
-to an empty table, then that process will fail.
-
-#### Defined in
-
-[embedding/embedding_function.ts:42](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L42)
-
-___
-
-### excludeSource
-
-• `Optional` **excludeSource**: `boolean`
-
-Should the source column be excluded from the resulting table
-
-By default the source column is included.  Set this to true and
-only the embedding will be stored.
-
-#### Defined in
-
-[embedding/embedding_function.ts:57](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L57)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Defined in
-
-[embedding/embedding_function.ts:24](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L24)

docs/src/js/modules.md

@@ -1,209 +0,0 @@
-[@lancedb/lancedb](README.md) / Exports
-
-# @lancedb/lancedb
-
-## Table of contents
-
-### Namespaces
-
-- [embedding](modules/embedding.md)
-
-### Enumerations
-
-- [WriteMode](enums/WriteMode.md)
-
-### Classes
-
-- [Connection](classes/Connection.md)
-- [Index](classes/Index.md)
-- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
-- [Query](classes/Query.md)
-- [QueryBase](classes/QueryBase.md)
-- [RecordBatchIterator](classes/RecordBatchIterator.md)
-- [Table](classes/Table.md)
-- [VectorColumnOptions](classes/VectorColumnOptions.md)
-- [VectorQuery](classes/VectorQuery.md)
-
-### Interfaces
-
-- [AddColumnsSql](interfaces/AddColumnsSql.md)
-- [AddDataOptions](interfaces/AddDataOptions.md)
-- [ColumnAlteration](interfaces/ColumnAlteration.md)
-- [ConnectionOptions](interfaces/ConnectionOptions.md)
-- [CreateTableOptions](interfaces/CreateTableOptions.md)
-- [ExecutableQuery](interfaces/ExecutableQuery.md)
-- [IndexConfig](interfaces/IndexConfig.md)
-- [IndexOptions](interfaces/IndexOptions.md)
-- [IvfPqOptions](interfaces/IvfPqOptions.md)
-- [TableNamesOptions](interfaces/TableNamesOptions.md)
-- [UpdateOptions](interfaces/UpdateOptions.md)
-- [WriteOptions](interfaces/WriteOptions.md)
-
-### Type Aliases
-
-- [Data](modules.md#data)
-
-### Functions
-
-- [connect](modules.md#connect)
-- [makeArrowTable](modules.md#makearrowtable)
-
-## Type Aliases
-
-### Data
-
-Ƭ **Data**: `Record`\<`string`, `unknown`\>[] \| `ArrowTable`
-
-Data type accepted by NodeJS SDK
-
-#### Defined in
-
-[arrow.ts:40](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L40)
-
-## Functions
-
-### connect
-
-▸ **connect**(`uri`, `opts?`): `Promise`\<[`Connection`](classes/Connection.md)\>
-
-Connect to a LanceDB instance at the given URI.
-
-Accpeted formats:
-
-- `/path/to/database` - local database
-- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
-- `db://host:port` - remote database (LanceDB cloud)
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. If the database uri starts with `db://` then it connects to a remote database. |
-| `opts?` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> | - |
-
-#### Returns
-
-`Promise`\<[`Connection`](classes/Connection.md)\>
-
-**`See`**
-
-[ConnectionOptions](interfaces/ConnectionOptions.md) for more details on the URI format.
-
-#### Defined in
-
-[index.ts:62](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/index.ts#L62)
-
-___
-
-### makeArrowTable
-
-▸ **makeArrowTable**(`data`, `options?`): `ArrowTable`
-
-An enhanced version of the makeTable function from Apache Arrow
-that supports nested fields and embeddings columns.
-
-(typically you do not need to call this function.  It will be called automatically
-when creating a table or adding data to it)
-
-This function converts an array of Record<String, any> (row-major JS objects)
-to an Arrow Table (a columnar structure)
-
-Note that it currently does not support nulls.
-
-If a schema is provided then it will be used to determine the resulting array
-types.  Fields will also be reordered to fit the order defined by the schema.
-
-If a schema is not provided then the types will be inferred and the field order
-will be controlled by the order of properties in the first record.  If a type
-is inferred it will always be nullable.
-
-If the input is empty then a schema must be provided to create an empty table.
-
-When a schema is not specified then data types will be inferred.  The inference
-rules are as follows:
-
- - boolean => Bool
- - number => Float64
- - String => Utf8
- - Buffer => Binary
- - Record<String, any> => Struct
- - Array<any> => List
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] |
-| `options?` | `Partial`\<[`MakeArrowTableOptions`](classes/MakeArrowTableOptions.md)\> |
-
-#### Returns
-
-`ArrowTable`
-
-**`Example`**
-
-```ts
-import { fromTableToBuffer, makeArrowTable } from "../arrow";
-import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
-
-const schema = new Schema([
-  new Field("a", new Int32()),
-  new Field("b", new Float32()),
-  new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
- ]);
- const table = makeArrowTable([
-   { a: 1, b: 2, c: [1, 2, 3] },
-   { a: 4, b: 5, c: [4, 5, 6] },
-   { a: 7, b: 8, c: [7, 8, 9] },
- ], { schema });
-```
-
-By default it assumes that the column named `vector` is a vector column
-and it will be converted into a fixed size list array of type float32.
-The `vectorColumns` option can be used to support other vector column
-names and data types.
-
-```ts
-
-const schema = new Schema([
-   new Field("a", new Float64()),
-   new Field("b", new Float64()),
-   new Field(
-     "vector",
-     new FixedSizeList(3, new Field("item", new Float32()))
-   ),
- ]);
- const table = makeArrowTable([
-   { a: 1, b: 2, vector: [1, 2, 3] },
-   { a: 4, b: 5, vector: [4, 5, 6] },
-   { a: 7, b: 8, vector: [7, 8, 9] },
- ]);
- assert.deepEqual(table.schema, schema);
-```
-
-You can specify the vector column types and names using the options as well
-
-```typescript
-
-const schema = new Schema([
-   new Field('a', new Float64()),
-   new Field('b', new Float64()),
-   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
- ]);
-const table = makeArrowTable([
-   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
- ], {
-   vectorColumns: {
-     vec1: { type: new Float16() },
-     vec2: { type: new Float16() }
-   }
- }
-assert.deepEqual(table.schema, schema)
-```
-
-#### Defined in
-
-[arrow.ts:197](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L197)

docs/src/js/modules/embedding.md

@@ -1,45 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / embedding
-
-# Namespace: embedding
-
-## Table of contents
-
-### Classes
-
-- [OpenAIEmbeddingFunction](../classes/embedding.OpenAIEmbeddingFunction.md)
-
-### Interfaces
-
-- [EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md)
-
-### Functions
-
-- [isEmbeddingFunction](embedding.md#isembeddingfunction)
-
-## Functions
-
-### isEmbeddingFunction
-
-▸ **isEmbeddingFunction**\<`T`\>(`value`): value is EmbeddingFunction\<T\>
-
-Test if the input seems to be an embedding function
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `value` | `unknown` |
-
-#### Returns
-
-value is EmbeddingFunction\<T\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:66](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L66)

docs/src/migration.md

@@ -1,76 +0,0 @@
-# Rust-backed Client Migration Guide
-
-In an effort to ensure all clients have the same set of capabilities we have begun migrating the
-python and node clients onto a common Rust base library. In python, this new client is part of
-the same lancedb package, exposed as an asynchronous client. Once the asynchronous client has
-reached full functionality we will begin migrating the synchronous library to be a thin wrapper
-around the asynchronous client.
-
-This guide describes the differences between the two APIs and will hopefully assist users
-that would like to migrate to the new API.
-
-## Closeable Connections
-
-The Connection now has a `close` method. You can call this when
-you are done with the connection to eagerly free resources. Currently
-this is limited to freeing/closing the HTTP connection for remote
-connections. In the future we may add caching or other resources to
-native connections so this is probably a good practice even if you
-aren't using remote connections.
-
-In addition, the connection can be used as a context manager which may
-be a more convenient way to ensure the connection is closed.
-
-```python
-import lancedb
-
-async def my_async_fn():
-    with await lancedb.connect_async("my_uri") as db:
-        print(await db.table_names())
-```
-
-It is not mandatory to call the `close` method. If you do not call it
-then the connection will be closed when the object is garbage collected.
-
-## Closeable Table
-
-The Table now also has a `close` method, similar to the connection. This
-can be used to eagerly free the cache used by a Table object. Similar to
-the connection, it can be used as a context manager and it is not mandatory
-to call the `close` method.
-
-### Changes to Table APIs
-
-- Previously `Table.schema` was a property. Now it is an async method.
-- The method `Table.__len__` was removed and `len(table)` will no longer
-  work. Use `Table.count_rows` instead.
-
-### Creating Indices
-
-The `Table.create_index` method is now used for creating both vector indices
-and scalar indices. It currently requires a column name to be specified (the
-column to index). Vector index defaults are now smarter and scale better with
-the size of the data.
-
-To specify index configuration details you will need to specify which kind of
-index you are using.
-
-### Querying
-
-The `Table.search` method has been renamed to `AsyncTable.vector_search` for
-clarity.
-
-## Features not yet supported
-
-The following features are not yet supported by the asynchronous API. However,
-we plan to support them soon.
-
-- You cannot specify an embedding function when creating or opening a table.
-  You must calculate embeddings yourself if using the asynchronous API
-- The merge insert operation is not supported in the asynchronous API
-- Cleanup / compact / optimize indices are not supported in the asynchronous API
-- add / alter columns is not supported in the asynchronous API
-- The asynchronous API does not yet support any full text search or reranking
-  search
-- Remote connections to LanceDb Cloud are not yet supported.
-- The method Table.head is not yet supported.

docs/src/python/duckdb.md

@@ -24,8 +24,7 @@ data = [
 table = db.create_table("pd_table", data=data)
 ```
 
-The `to_lance` method converts the LanceDB table to a `LanceDataset`, which is accessible to DuckDB through the Arrow compatibility layer.
-To query the resulting Lance dataset in DuckDB, all you need to do is reference the dataset by the same name in your SQL query.
+To query the table, first call `to_lance` to convert the table to a "dataset", which is an object that can be queried by DuckDB. Then all you need to do is reference that dataset by the same name in your SQL query.
 
 ```python
 import duckdb

docs/src/python/python.md

@@ -8,31 +8,22 @@ This section contains the API reference for the OSS Python API.
 pip install lancedb
 ```
 
-The following methods describe the synchronous API client. There
-is also an [asynchronous API client](#connections-asynchronous).
-
-## Connections (Synchronous)
+## Connection
 
 ::: lancedb.connect
 
 ::: lancedb.db.DBConnection
 
-## Tables (Synchronous)
+## Table
 
 ::: lancedb.table.Table
 
-## Querying (Synchronous)
+## Querying
 
 ::: lancedb.query.Query
 
 ::: lancedb.query.LanceQueryBuilder
 
-::: lancedb.query.LanceVectorQueryBuilder
-
-::: lancedb.query.LanceFtsQueryBuilder
-
-::: lancedb.query.LanceHybridQueryBuilder
-
 ## Embeddings
 
 ::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
@@ -71,60 +62,10 @@ is also an [asynchronous API client](#connections-asynchronous).
 
 ## Integrations
 
-## Pydantic
+### Pydantic
 
 ::: lancedb.pydantic.pydantic_to_schema
 
 ::: lancedb.pydantic.vector
 
 ::: lancedb.pydantic.LanceModel
-
-## Reranking
-
-::: lancedb.rerankers.linear_combination.LinearCombinationReranker
-
-::: lancedb.rerankers.cohere.CohereReranker
-
-::: lancedb.rerankers.colbert.ColbertReranker
-
-::: lancedb.rerankers.cross_encoder.CrossEncoderReranker
-
-::: lancedb.rerankers.openai.OpenaiReranker
-
-## Connections (Asynchronous)
-
-Connections represent a connection to a LanceDb database and
-can be used to create, list, or open tables.
-
-::: lancedb.connect_async
-
-::: lancedb.db.AsyncConnection
-
-## Tables (Asynchronous)
-
-Table hold your actual data as a collection of records / rows.
-
-::: lancedb.table.AsyncTable
-
-## Indices (Asynchronous)
-
-Indices can be created on a table to speed up queries. This section
-lists the indices that LanceDb supports.
-
-::: lancedb.index.BTree
-
-::: lancedb.index.IvfPq
-
-## Querying (Asynchronous)
-
-Queries allow you to return data from your database. Basic queries can be
-created with the [AsyncTable.query][lancedb.table.AsyncTable.query] method
-to return the entire (typically filtered) table. Vector searches return the
-rows nearest to a query vector and can be created with the
-[AsyncTable.vector_search][lancedb.table.AsyncTable.vector_search] method.
-
-::: lancedb.query.AsyncQueryBase
-
-::: lancedb.query.AsyncQuery
-
-::: lancedb.query.AsyncVectorQuery

docs/src/reranking/cohere.md

@@ -1,75 +0,0 @@
-# Cohere Reranker
-
-This re-ranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
-
-
-!!! note
-    Supported Query Types: Hybrid, Vector, FTS
-
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import CohereReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-tbl.add(data)
-reranker = CohereReranker(api_key="key")
-
-# Run vector search with a reranker
-result = tbl.search("hello").rerank(reranker=reranker).to_list() 
-
-# Run FTS search with a reranker
-result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text", replace=True)
-result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
-
-```
-
-Accepted Arguments
-----------------
-| Argument | Type | Default | Description |
-| --- | --- | --- | --- |
-| `model_name` | `str` | `"rerank-english-v2.0"` | The name of the reranker model to use. Available cohere models are: rerank-english-v2.0, rerank-multilingual-v2.0 |
-| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
-| `api_key` | `str` | `None` | The API key for the Cohere API. If not provided, the `COHERE_API_KEY` environment variable is used. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
-
-
-
-## Supported Scores for each query type
-You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
-
-### Hybrid Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
-
-### Vector Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
-
-### FTS Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/colbert.md

@@ -1,71 +0,0 @@
-# ColBERT Reranker
-
-This re-ranker uses ColBERT model to rerank the search results. You can use this re-ranker by passing `ColbertReranker()` to the `rerank()` method. 
-!!! note
-    Supported Query Types: Hybrid, Vector, FTS
-
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import ColbertReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-tbl.add(data)
-reranker = ColbertReranker()
-
-# Run vector search with a reranker
-result = tbl.search("hello").rerank(reranker=reranker).to_list() 
-
-# Run FTS search with a reranker
-result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text", replace=True)
-result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
-
-```
-
-Accepted Arguments
-----------------
-| Argument | Type | Default | Description |
-| --- | --- | --- | --- |
-| `model_name` | `str` | `"colbert-ir/colbertv2.0"` | The name of the reranker model to use.|
-| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
-
-
-## Supported Scores for each query type
-You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
-
-### Hybrid Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
-
-### Vector Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
-
-### FTS Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/cross_encoder.md

@@ -1,70 +0,0 @@
-# Cross Encoder Reranker
-
-This re-ranker uses Cross Encoder models from sentence-transformers to rerank the search results. You can use this re-ranker by passing `CrossEncoderReranker()` to the `rerank()` method. 
-!!! note
-    Supported Query Types: Hybrid, Vector, FTS
-
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import CrossEncoderReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-tbl.add(data)
-reranker = CrossEncoderReranker()
-
-# Run vector search with a reranker
-result = tbl.search("hello").rerank(reranker=reranker).to_list() 
-
-# Run FTS search with a reranker
-result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text", replace=True)
-result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
-
-```
-
-Accepted Arguments
-----------------
-| Argument | Type | Default | Description |
-| --- | --- | --- | --- |
-| `model_name` | `str` | `""cross-encoder/ms-marco-TinyBERT-L-6"` | The name of the reranker model to use.|
-| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `device` | `str` | `None` | The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu". |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
-
-## Supported Scores for each query type
-You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
-
-### Hybrid Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
-
-### Vector Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
-
-### FTS Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/reranking/custom_reranker.md

@@ -1,88 +0,0 @@
-## Building Custom Rerankers
-You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Optionally, you can also implement the `rerank_vector()` and `rerank_fts()` methods if you want to support reranking for vector and FTS search separately.
-Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
-
-The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
-
-```python
-
-from lancedb.rerankers import Reranker
-import pyarrow as pa
-
-class MyReranker(Reranker):
-    def __init__(self, param1, param2, ..., return_score="relevance"):
-        super().__init__(return_score)
-        self.param1 = param1
-        self.param2 = param2
-
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
-        # Use the built-in merging function
-        combined_result = self.merge_results(vector_results, fts_results)
-
-        # Do something with the combined results
-        # ...
-
-        # Return the combined results
-        return combined_result
-
-    def rerank_vector(self, query: str, vector_results: pa.Table):
-        # Do something with the vector results
-        # ...
-
-        # Return the vector results
-        return vector_results
-
-    def rerank_fts(self, query: str, fts_results: pa.Table):
-        # Do something with the FTS results
-        # ...
-
-        # Return the FTS results
-        return fts_results
-
-```
-
-### Example of a Custom Reranker
-For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
-
-```python
-
-from typing import List, Union
-import pandas as pd
-from lancedb.rerankers import CohereReranker
-
-class ModifiedCohereReranker(CohereReranker):
-    def __init__(self, filters: Union[str, List[str]], **kwargs):
-        super().__init__(**kwargs)
-        filters = filters if isinstance(filters, list) else [filters]
-        self.filters = filters
-
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
-        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
-        df = combined_result.to_pandas()
-        for filter in self.filters:
-            df = df.query("not text.str.contains(@filter)")
-
-        return pa.Table.from_pandas(df)
-
-    def rerank_vector(self, query: str, vector_results: pa.Table)-> pa.Table:
-        vector_results = super().rerank_vector(query, vector_results)
-        df = vector_results.to_pandas()
-        for filter in self.filters:
-            df = df.query("not text.str.contains(@filter)")
-
-        return pa.Table.from_pandas(df)
-
-    def rerank_fts(self, query: str, fts_results: pa.Table)-> pa.Table:
-        fts_results = super().rerank_fts(query, fts_results)
-        df = fts_results.to_pandas()
-        for filter in self.filters:
-            df = df.query("not text.str.contains(@filter)")
-
-        return pa.Table.from_pandas(df)
-
-```
-
-!!! tip
-    The `vector_results` and `fts_results` are pyarrow tables. Lean more about pyarrow tables [here](https://arrow.apache.org/docs/python). It can be convered to other data types like pandas dataframe, pydict, pylist etc.
-
-    For example, You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/reranking/index.md

@@ -1,60 +0,0 @@
-Reranking is the process of reordering a list of items based on some criteria. In the context of search, reranking is used to reorder the search results returned by a search engine based on some criteria. This can be useful when the initial ranking of the search results is not satisfactory or when the user has provided additional information that can be used to improve the ranking of the search results.
-
-LanceDB comes with some built-in rerankers. Some of the rerankers that are available in LanceDB are:
-
-| Reranker | Description | Supported Query Types |
-| --- | --- | --- |
-| `LinearCombinationReranker` | Reranks search results based on a linear combination of FTS and vector search scores | Hybrid |
-| `CohereReranker` | Uses cohere Reranker API to rerank results | Vector, FTS, Hybrid |
-| `CrossEncoderReranker` | Uses a cross-encoder model to rerank search results | Vector, FTS, Hybrid |
-| `ColbertReranker` | Uses a colbert model to rerank search results | Vector, FTS, Hybrid |
-| `OpenaiReranker`(Experimental) | Uses OpenAI's chat model to rerank search results | Vector, FTS, Hybrid |
-
-
-## Using a Reranker
-Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import CohereReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", data)
-reranker = CohereReranker(api_key="your_api_key")
-
-# Run vector search with a reranker
-result = tbl.query("hello").rerank(reranker).to_list() 
-
-# Run FTS search with a reranker
-result = tbl.query("hello", query_type="fts").rerank(reranker).to_list()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text")
-result = tbl.query("hello", query_type="hybrid").rerank(reranker).to_list()
-```
-
-## Available Rerankers
-LanceDB comes with some built-in rerankers. Here are some of the rerankers that are available in LanceDB:
-
-- [Cohere Reranker](./cohere.md)
-- [Cross Encoder Reranker](./cross_encoder.md)
-- [ColBERT Reranker](./colbert.md)
-- [OpenAI Reranker](./openai.md)
-- [Linear Combination Reranker](./linear_combination.md)
-
-## Creating Custom Rerankers
-
-LanceDB also you to create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. This is covered in more detail in the [Creating Custom Rerankers](./custom_reranker.md) section.

docs/src/reranking/linear_combination.md

@@ -1,52 +0,0 @@
-# Linear Combination Reranker
-
-This is the default re-ranker used by LanceDB hybrid search. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
-
-!!! note
-    Supported Query Types: Hybrid
-
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import LinearCombinationReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-tbl.add(data)
-reranker = LinearCombinationReranker()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text", replace=True)
-result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
-
-```
-
-Accepted Arguments
-----------------
-| Argument | Type | Default | Description |
-| --- | --- | --- | --- |
-| `weight` | `float` | `0.7` | The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score. |
-
-
-## Supported Scores for each query type
-You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
-
-### Hybrid Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_distance`) |

docs/src/reranking/openai.md

@@ -1,73 +0,0 @@
-# OpenAI Reranker (Experimental)
-
-This re-ranker uses OpenAI chat model to rerank the search results. You can use this re-ranker by passing `OpenAI()` to the `rerank()` method. 
-!!! note
-    Supported Query Types: Hybrid, Vector, FTS
-
-!!! warning
-    This re-ranker is experimental. OpenAI doesn't have a dedicated reranking model, so we are using the chat model for reranking. 
-
-```python
-import numpy
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.rerankers import OpenaiReranker
-
-embedder = get_registry().get("sentence-transformers").create()
-db = lancedb.connect("~/.lancedb")
-
-class Schema(LanceModel):
-    text: str = embedder.SourceField()
-    vector: Vector(embedder.ndims()) = embedder.VectorField()
-
-data = [
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-    ]
-tbl = db.create_table("test", schema=Schema, mode="overwrite")
-tbl.add(data)
-reranker = OpenaiReranker()
-
-# Run vector search with a reranker
-result = tbl.search("hello").rerank(reranker=reranker).to_list() 
-
-# Run FTS search with a reranker
-result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
-
-# Run hybrid search with a reranker
-tbl.create_fts_index("text", replace=True)
-result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
-
-```
-
-Accepted Arguments
-----------------
-| Argument | Type | Default | Description |
-| --- | --- | --- | --- |
-| `model_name` | `str` | `"gpt-4-turbo-preview"` | The name of the reranker model to use.|
-| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
-| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
-| `api_key` | str | `None` | The API key to use. If None, will use the OPENAI_API_KEY environment variable.
-
-
-## Supported Scores for each query type
-You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
-
-### Hybrid Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
-
-### Vector Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
-
-### FTS Search
-|`return_score`| Status | Description |
-| --- | --- | --- |
-| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
-| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/search.md

@@ -22,7 +22,7 @@ Currently, LanceDB supports the following metrics:
 ## Exhaustive search (kNN)
 
 If you do not create a vector index, LanceDB exhaustively scans the _entire_ vector space
-and computes the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
+and compute the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
 
 <!-- Setup Code
 ```python
@@ -85,7 +85,7 @@ To perform scalable vector retrieval with acceptable latencies, it's common to b
 While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of
 an ANN search means that using an index often involves a trade-off between recall and latency.
 
-See the [IVF_PQ index](./concepts/index_ivfpq.md) for a deeper description of how `IVF_PQ`
+See the [IVF_PQ index](./concepts/index_ivfpq.md.md) for a deeper description of how `IVF_PQ`
 indexes work in LanceDB.
 
 ## Output search results
@@ -184,3 +184,4 @@ Let's create a LanceDB table with a nested schema:
 
     Note that in this case the extra `_distance` field is discarded since
     it's not part of the LanceSchema.
+

docs/src/sql.md

@@ -66,7 +66,6 @@ Currently, Lance supports a growing list of SQL expressions.
 - `LIKE`, `NOT LIKE`
 - `CAST`
 - `regexp_match(column, pattern)`
-- [DataFusion Functions](https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html)
 
 For example, the following filter string is acceptable:
 

docs/test/md_testing.py

@@ -1,5 +1,5 @@
 import glob
-from typing import Iterator, List
+from typing import Iterator
 from pathlib import Path
 
 glob_string = "../src/**/*.md"
@@ -15,7 +15,6 @@ excluded_globs = [
     "../src/ann_indexes.md",
     "../src/basic.md",
     "../src/hybrid_search/hybrid_search.md",
-    "../src/reranking/*.md",
 ]
 
 python_prefix = "py"
@@ -51,24 +50,11 @@ def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
                 yield line[strip_length:]
 
 
-def wrap_async(lines: List[str]) -> List[str]:
-    # Indent all the lines
-    lines = ["    " + line for line in lines]
-    # Put all lines in `async def main():`
-    lines = ["async def main():\n"] + lines
-    # Put `import asyncio\n asyncio.run(main())` at the end
-    lines = lines + ["\n", "import asyncio\n", "asyncio.run(main())\n"]
-    return lines
-
-
 for file in filter(lambda file: file not in excluded_files, files):
     with open(file, "r") as f:
         lines = list(yield_lines(iter(f), "```", "```"))
 
     if len(lines) > 0:
-        if any("await" in line for line in lines):
-            lines = wrap_async(lines)
-
         print(lines)
         out_path = (
             Path(python_folder)

node/.eslintrc.js

@@ -13,10 +13,5 @@ module.exports = {
   },
   rules: {
     "@typescript-eslint/method-signature-style": "off",
-    "@typescript-eslint/quotes": "off",
-    "@typescript-eslint/semi": "off",
-    "@typescript-eslint/explicit-function-return-type": "off",
-    "@typescript-eslint/space-before-function-paren": "off",
-    "@typescript-eslint/indent": "off",
   }
 }

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.17",
+  "version": "0.4.11",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.17",
+      "version": "0.4.11",
       "cpu": [
         "x64",
         "arm64"
@@ -18,7 +18,9 @@
         "win32"
       ],
       "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
         "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
         "axios": "^1.4.0"
       },
       "devDependencies": {
@@ -31,7 +33,6 @@
         "@types/temp": "^0.9.1",
         "@types/uuid": "^9.0.3",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "apache-arrow-old": "npm:apache-arrow@13.0.0",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
         "chai-as-promised": "^7.1.1",
@@ -52,15 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.17",
-        "@lancedb/vectordb-darwin-x64": "0.4.17",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.17",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.17",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.17"
-      },
-      "peerDependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "apache-arrow": "^14.0.2"
+        "@lancedb/vectordb-darwin-arm64": "0.4.11",
+        "@lancedb/vectordb-darwin-x64": "0.4.11",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -96,7 +93,6 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz",
       "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==",
-      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -113,8 +109,7 @@
     "node_modules/@apache-arrow/ts/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
-      "peer": true
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
     },
     "node_modules/@cargo-messages/android-arm-eabi": {
       "version": "0.0.160",
@@ -333,6 +328,66 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.11.tgz",
+      "integrity": "sha512-JDOKmFnuJPFkA7ZmrzBJolROwSjWr7yMvAbi40uLBc25YbbVezodd30u2EFtIwWwtk1GqNYRZ49FZOElKYeC/Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-darwin-x64": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.11.tgz",
+      "integrity": "sha512-iy6r+8tp2v1EFgJV52jusXtxgO6NY6SkpOdX41xPqN2mQWMkfUAR9Xtks1mgknjPOIKH4MRc8ZS0jcW/UWmilQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.11.tgz",
+      "integrity": "sha512-5K6IVcTMuH0SZBjlqB5Gg39WC889FpTwIWKufxzQMMXrzxo5J3lKUHVoR28RRlNhDF2d9kZXBEyCpIfDFsV9iQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.11.tgz",
+      "integrity": "sha512-hF9ZChsdqKqqnivOzd9mE7lC3PmhZadXtwThi2RrsPiOLoEaGDfmr6Ni3amVQnB3bR8YEJtTxdQxe0NC4uW/8g==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.4.11",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.11.tgz",
+      "integrity": "sha512-0+9ut1ccKoqIyGxsVixwx3771Z+DXpl5WfSmOeA8kf3v3jlOg2H+0YUahiXLDid2ju+yeLPrAUYm7A1gKHVhew==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
@@ -893,7 +948,6 @@
       "version": "14.0.2",
       "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz",
       "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==",
-      "peer": true,
       "dependencies": {
         "@types/command-line-args": "5.2.0",
         "@types/command-line-usage": "5.0.2",
@@ -910,39 +964,10 @@
         "arrow2csv": "bin/arrow2csv.js"
       }
     },
-    "node_modules/apache-arrow-old": {
-      "name": "apache-arrow",
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-13.0.0.tgz",
-      "integrity": "sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==",
-      "dev": true,
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/apache-arrow-old/node_modules/@types/node": {
-      "version": "20.3.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
-      "dev": true
-    },
     "node_modules/apache-arrow/node_modules/@types/node": {
       "version": "20.3.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz",
-      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==",
-      "peer": true
+      "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ=="
     },
     "node_modules/arg": {
       "version": "4.1.3",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.4.17",
+  "version": "0.4.11",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -41,7 +41,6 @@
     "@types/temp": "^0.9.1",
     "@types/uuid": "^9.0.3",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
-    "apache-arrow-old": "npm:apache-arrow@13.0.0",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
@@ -88,10 +87,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.17",
-    "@lancedb/vectordb-darwin-x64": "0.4.17",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.17",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.17",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.17"
+    "@lancedb/vectordb-darwin-arm64": "0.4.11",
+    "@lancedb/vectordb-darwin-x64": "0.4.11",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
   }
 }