[python] Bump version: 0.5.3 → 0.5.4

Closes #940

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.1
+current_version = 0.4.8
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.cargo/config.toml

@@ -0,0 +1,34 @@
+[profile.release]
+lto = "fat"
+codegen-units = 1
+
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+# Prioritize compile time over runtime performance
+codegen-units = 16
+lto = "thin"
+
+[target.'cfg(all())']
+rustflags = [
+    "-Wclippy::all",
+    "-Wclippy::style",
+    "-Wclippy::fallible_impl_from",
+    "-Wclippy::manual_let_else",
+    "-Wclippy::redundant_pub_crate",
+    "-Wclippy::string_add_assign",
+    "-Wclippy::string_add",
+    "-Wclippy::string_lit_as_bytes",
+    "-Wclippy::string_to_string",
+    "-Wclippy::use_self",
+    "-Dclippy::cargo",
+    "-Dclippy::dbg_macro",
+    # not too much we can do to avoid multiple crate versions
+    "-Aclippy::multiple-crate-versions",
+]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]

.github/workflows/cargo-publish.yml

@@ -16,7 +16,7 @@ jobs:
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust

.github/workflows/docs.yml

@@ -27,9 +27,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"
@@ -42,7 +42,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - uses: Swatinem/rust-cache@v2
@@ -62,8 +62,9 @@ jobs:
         run: |
           npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
+        working-directory: docs
         run: |
-          PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
+          PYTHONPATH=. mkdocs build
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact
@@ -72,4 +73,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -18,24 +18,20 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-minor-version: [ "11" ]
-        os: ["ubuntu-22.04"]
+    runs-on: "ubuntu-latest"
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.${{ matrix.python-minor-version }}
+        python-version: 3.11
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
     - name: Build Python
@@ -52,42 +48,42 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
+    timeout-minutes: 45
     strategy:
-      matrix:
-        node-version: [ "18" ]
-        os: ["ubuntu-22.04"]
+      fail-fast: false
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.node-version }}
+        node-version: 20
     - name: Install dependecies needed for ubuntu
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
-    - name: Install node dependencies
-      run: |
-        cd docs/test
-        npm install 
     - name: Rust cache
       uses: swatinem/rust-cache@v2
-    - name: Install LanceDB
+    - name: Install node dependencies
       run: |
-        cd docs/test/node_modules/vectordb 
+        sudo swapoff -a
+        sudo fallocate -l 8G /swapfile
+        sudo chmod 600 /swapfile
+        sudo mkswap /swapfile
+        sudo swapon /swapfile
+        sudo swapon --show
+        cd node
         npm ci
         npm run build-release
-        npm run tsc
-    - name: Create test files
-      run: |
-        cd docs/test
-        node md_testing.js
+        cd ../docs
+        npm install
     - name: Test
+      env:
+        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
+        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
       run: |
-        cd docs/test/node
-        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done
+        cd docs
+        npm t

.github/workflows/make-release-commit.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out main
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false
@@ -37,10 +37,10 @@ jobs:
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Bump version, create tag and commit
         run: |
           pip install bump2version

.github/workflows/node.yml

@@ -18,7 +18,9 @@ concurrency:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
+  #
+  # Use native CPU to accelerate tests if possible, especially for f16
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
@@ -30,13 +32,13 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 20
         cache: 'npm'
         cache-dependency-path: node/package-lock.json
     - name: Lint
@@ -48,14 +50,14 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        node-version: [ "16", "18" ]
+        node-version: [ "18", "20" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -72,7 +74,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz
@@ -88,13 +89,13 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 20
         cache: 'npm'
         cache-dependency-path: node/package-lock.json
     - uses: Swatinem/rust-cache@v2
@@ -103,7 +104,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz
@@ -128,13 +128,13 @@ jobs:
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 20
         cache: 'npm'
         cache-dependency-path: node/package-lock.json
     - name: start local stack
@@ -157,7 +157,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz

.github/workflows/nodejs.yml

@@ -0,0 +1,114 @@
+name: NodeJS (NAPI)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - nodejs/**
+      - .github/workflows/nodejs.yml
+      - docker-compose.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: nodejs/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Lint
+      run: |
+        cargo fmt --all -- --check
+        cargo clippy --all --all-features -- -D warnings
+        npm ci
+        npm run lint
+  linux:
+    name: Linux (NodeJS ${{ matrix.node-version }})
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        node-version: [ "18", "20" ]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Test
+      run: npm run test
+  macos:
+    timeout-minutes: 30
+    runs-on: "macos-14"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        brew install protobuf
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Test
+      run: |
+        npm run test
+

.github/workflows/npm-publish.yml

@@ -15,7 +15,7 @@ jobs:
         working-directory: node
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - uses: actions/setup-node@v3
         with:
           node-version: 20
@@ -45,13 +45,13 @@ jobs:
             runner: macos-13
           - arch: aarch64-apple-darwin
             # xlarge is implicitly arm64.
-            runner: macos-13-xlarge
+            runner: macos-14
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install system dependencies
         run: brew install protobuf
       - name: Install npm dependencies
@@ -66,7 +66,7 @@ jobs:
           name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
-         
+
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -83,7 +83,7 @@ jobs:
             runner: buildjet-4vcpu-ubuntu-2204-arm
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build Linux Artifacts
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
@@ -104,7 +104,7 @@ jobs:
         target: [x86_64-pc-windows-msvc]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -154,7 +154,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false

.github/workflows/pypi-publish.yml

@@ -14,9 +14,9 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.8"
       - name: Build distribution

.github/workflows/python-make-release-commit.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out main
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: main
         persist-credentials: false
@@ -37,10 +37,10 @@ jobs:
       run: |
         git config user.name 'Lance Release'
         git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.11"
     - name: Bump version, create tag and commit
       working-directory: python
       run: |

.github/workflows/python.yml

@@ -18,19 +18,19 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        python-minor-version: [ "8", "9", "10", "11" ]
+        python-minor-version: [ "8", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
@@ -38,8 +38,10 @@ jobs:
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock ruff
-    - name: Lint
+    - name: Format check
       run: ruff format --check .
+    - name: Lint
+      run: ruff .
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
@@ -49,11 +51,11 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        config: 
+        config:
           - name: x86 Mac
             runner: macos-13
           - name: Arm Mac
-            runner: macos-13-xlarge
+            runner: macos-14
           - name: x86 Windows
             runner: windows-latest
     runs-on: "${{ matrix.config.runner }}"
@@ -62,19 +64,19 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.11"
     - name: Install lancedb
       run: |
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black
+        pip install pytest pytest-mock
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
   pydantic1x:
@@ -85,12 +87,12 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Install lancedb

.github/workflows/rust.yml

@@ -32,7 +32,7 @@ jobs:
         shell: bash
         working-directory: rust
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           fetch-depth: 0
           lfs: true
@@ -55,7 +55,7 @@ jobs:
         shell: bash
         working-directory: rust
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           fetch-depth: 0
           lfs: true
@@ -70,18 +70,20 @@ jobs:
       run: cargo build --all-features
     - name: Run tests
       run: cargo test --all-features
+    - name: Run examples
+      run: cargo run --example simple
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+        mac-runner: [ "macos-13", "macos-14" ]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
         working-directory: rust
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
             fetch-depth: 0
             lfs: true
@@ -99,7 +101,7 @@ jobs:
   windows:
     runs-on: windows-2022
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust

.github/workflows/update_package_lock_run.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false

.gitignore

@@ -29,8 +29,9 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+dist
 
 ## Rust
 target
 
-Cargo.lock
+Cargo.lock

Cargo.toml

@@ -1,28 +1,40 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb"]
+members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
+[workspace.package]
+edition = "2021"
+authors = ["LanceDB Devs <dev@lancedb.com>"]
+license = "Apache-2.0"
+repository = "https://github.com/lancedb/lancedb"
+description = "Serverless, low-latency vector database for AI applications"
+keywords = ["lancedb", "lance", "database", "vector", "search"]
+categories = ["database-implementations"]
+
 [workspace.dependencies]
-lance = { "version" = "=0.9.1", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.1" }
-lance-linalg = { "version" = "=0.9.1" }
-lance-testing = { "version" = "=0.9.1" }
+lance = { "version" = "=0.9.15", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.9.15" }
+lance-linalg = { "version" = "=0.9.15" }
+lance-testing = { "version" = "=0.9.15" }
 # Note that this one does not include pyarrow
-arrow = { version = "49.0.0", optional = false }
-arrow-array = "49.0"
-arrow-data = "49.0"
-arrow-ipc = "49.0"
-arrow-ord = "49.0"
-arrow-schema = "49.0"
-arrow-arith = "49.0"
-arrow-cast = "49.0"
+arrow = { version = "50.0", optional = false }
+arrow-array = "50.0"
+arrow-data = "50.0"
+arrow-ipc = "50.0"
+arrow-ord = "50.0"
+arrow-schema = "50.0"
+arrow-arith = "50.0"
+arrow-cast = "50.0"
+async-trait = "0"
 chrono = "0.4.23"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
+futures = "0"
 log = "0.4"
-object_store = "0.8.0"
+object_store = "0.9.0"
 snafu = "0.7.4"
 url = "2"
+num-traits = "0.2"

README.md

@@ -51,12 +51,19 @@ npm install vectordb
 const lancedb = require('vectordb');
 const db = await lancedb.connect('data/sample-lancedb');
 
-const table = await db.createTable('vectors',
-      [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-       { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
+const table = await db.createTable({
+  name: 'vectors',
+  data:  [
+    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
+  ]
+})
 
 const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
+
+// You can also search for rows by specific criteria without involving a vector search.
+const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
 ```
 
 **Python**

docs/README.md

@@ -16,11 +16,29 @@ unreleased features.
 
 ### Building node module and create markdown files
 
-See [Javascript docs README](docs/src/javascript/README.md)
+See [Javascript docs README](./src/javascript/README.md)
 
 ### Build docs
 From LanceDB repo root:
 
 Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
 
-If successful, you should see a `docs/site` directory that you can verify locally.
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+### Run local server
+
+You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
+
+```bash
+cd docs
+mkdocs serve
+```
+
+### Run doctest for typescript example
+
+```bash
+cd lancedb/docs
+npm i
+npm run build
+npm run all
+```

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: LanceDB Docs
+site_name: LanceDB
 site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
@@ -9,20 +9,31 @@ theme:
   name: "material"
   logo: assets/logo.png
   favicon: assets/logo.png
+  palette:
+    # Palette toggle for light mode
+    - scheme: lancedb
+      primary: custom
+      toggle:
+        icon: material/weather-night
+        name: Switch to dark mode
+    # Palette toggle for dark mode
+    - scheme: slate
+      primary: custom
+      toggle:
+        icon: material/weather-sunny
+        name: Switch to light mode
   features:
     - content.code.copy
     - content.tabs.link
     - content.action.edit
     - toc.follow
-    - toc.integrate
+    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
     - navigation.footer
     - navigation.tracking
     - navigation.instant
-    - navigation.indexes
-    - navigation.expand
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -34,14 +45,12 @@ plugins:
     handlers:
       python:
         paths: [../python]
-        selection:
+        options:
           docstring_style: numpy
-        rendering:
           heading_level: 4
           show_source: true
           show_symbol_type_in_heading: true
           show_signature_annotations: true
-          show_root_heading: true
           members_order: source
         import:
           # for cross references
@@ -52,40 +61,51 @@ plugins:
 markdown_extensions:
 - admonition
 - footnotes
-- pymdownx.superfences
 - pymdownx.details
 - pymdownx.highlight:
     anchor_linenums: true
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
-- pymdownx.snippets
+- pymdownx.snippets:
+    base_path: ..
+    dedent_subsections: true
 - pymdownx.superfences
 - pymdownx.tabbed:
     alternate_style: true
 - md_in_html
+- attr_list
 
 nav:
 - Home:
-  - 🏢 Home: index.md
-  - 💡 Basics: basic.md
-  - 📚 Guides:
-    - Create Ingest Update Delete: guides/tables.md
+  - LanceDB: index.md
+  - 🏃🏼‍♂️ Quick start: basic.md
+  - 📚 Concepts:
+    - Vector search: concepts/vector_search.md
+    - Indexing: concepts/index_ivfpq.md
+    - Storage: concepts/storage.md
+    - Data management: concepts/data_management.md
+  - 🔨 Guides:
+    - Working with tables: guides/tables.md
+    - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
-    - SQL filters: sql.md
-    - Indexing: ann_indexes.md
+    - Full-text search: fts.md
+    - Hybrid search: hybrid_search.md
+    - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
-  - 🧬 Embeddings:
-    - embeddings/index.md
-    - Ingest Embedding Functions: embeddings/embedding_functions.md
+    - Configuring Storage: guides/storage.md
+  - 🧬 Managing embeddings:
+    - Overview: embeddings/index.md
+    - Explicit management: embeddings/embedding_explicit.md
+    - Implicit management: embeddings/embedding_functions.md
     - Available Functions: embeddings/default_embedding_functions.md
-    - Create Custom Embedding Functions: embeddings/api.md
-    - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
-    - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
-  - 🔍 Python full-text search: fts.md
+    - Custom Embedding Functions: embeddings/api.md
+    - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
+    - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔌 Integrations:
-    - integrations/index.md
-    - Pandas and PyArrow: python/arrow.md
+    - Tools and data formats: integrations/index.md
+    - Pandas and PyArrow: python/pandas_and_pyarrow.md
+    - Polars: python/polars_arrow.md
     - DuckDB: python/duckdb.md
     - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
     - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
@@ -93,43 +113,65 @@ nav:
     - Pydantic: python/pydantic.md
     - Voxel51: integrations/voxel51.md
     - PromptTools: integrations/prompttools.md
-  - 🐍 Python examples:
-    - examples/index.md
-    - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-    - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-    - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-    - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
-    - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-    - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-  - 🌐 Javascript examples:
-    - Examples: examples/index_js.md
-    - Serverless Website Chatbot: examples/serverless_website_chatbot.md
-    - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-    - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-  - ⚙️ CLI & Config: cli_config.md
+  - 🎯 Examples:
+    - Overview: examples/index.md
+    - 🐍 Python:
+      - Overview: examples/examples_python.md
+      - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+      - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+      - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+      - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
+      - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+      - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+    - 👾 JavaScript:
+      - Overview: examples/examples_js.md
+      - Serverless Website Chatbot: examples/serverless_website_chatbot.md
+      - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+      - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+  - 🔧 CLI & Config: cli_config.md
+  - 💭 FAQs: faq.md
+  - ⚙️ API reference:
+    - 🐍 Python: python/python.md
+    - 👾 JavaScript: javascript/modules.md
+    - 🦀 Rust: https://docs.rs/vectordb/latest/vectordb/
+  - ☁️ LanceDB Cloud:
+    - Overview: cloud/index.md
+    - API reference:
+      - 🐍 Python: python/saas-python.md
+      - 👾 JavaScript: javascript/saas-modules.md
 
-- Basics: basic.md
+
+- Quick start: basic.md
+- Concepts:
+    - Vector search: concepts/vector_search.md
+    - Indexing: concepts/index_ivfpq.md
+    - Storage: concepts/storage.md
+    - Data management: concepts/data_management.md
 - Guides:
-  - Create Ingest Update Delete: guides/tables.md
-  - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
-  - Versioning & Reproducibility: notebooks/reproducibility.ipynb
-- Embeddings:
-  - embeddings/index.md
-  - Ingest Embedding Functions: embeddings/embedding_functions.md
+    - Working with tables: guides/tables.md
+    - Building an ANN index: ann_indexes.md
+    - Vector Search: search.md
+    - Full-text search: fts.md
+    - Hybrid search: hybrid_search.md
+    - Filtering: sql.md
+    - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+    - Configuring Storage: guides/storage.md
+- Managing Embeddings:
+  - Overview: embeddings/index.md
+  - Explicit management: embeddings/embedding_explicit.md
+  - Implicit management: embeddings/embedding_functions.md
   - Available Functions: embeddings/default_embedding_functions.md
-  - Create Custom Embedding Functions: embeddings/api.md
-  - Example - Multi-lingual semantic search: notebooks/multi_lingual_example.ipynb
-  - Example - MultiModal CLIP Embeddings: notebooks/DisappearingEmbeddingFunction.ipynb
-- Python full-text search: fts.md
+  - Custom Embedding Functions: embeddings/api.md
+  - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
+  - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
 - Integrations:
-  - integrations/index.md
-  - Pandas and PyArrow: python/arrow.md
-  - DuckDB: python/duckdb.md
-  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-  - LangChain JS/TS 🦜️🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+  - Overview: integrations/index.md
+  - Pandas and PyArrow: python/pandas_and_pyarrow.md
+  - Polars: python/polars_arrow.md
+  - DuckDB : python/duckdb.md
+  - LangChain 🦜️🔗↗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+  - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
   - PromptTools: integrations/prompttools.md
@@ -141,18 +183,25 @@ nav:
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
 - Javascript examples:
-  - examples/index_js.md
+  - Overview: examples/examples_js.md
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-- API references:
-  - OSS Python API: python/python.md
-  - SaaS Python API: python/saas-python.md
-  - Javascript API: javascript/modules.md
-- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
+- API reference:
+  - Python: python/python.md
+  - Javascript: javascript/modules.md
+- LanceDB Cloud:
+    - Overview: cloud/index.md
+    - API reference:
+      - 🐍 Python: python/saas-python.md
+      - 👾 JavaScript: javascript/saas-modules.md
 
 extra_css:
   - styles/global.css
+  - styles/extra.css
+
+extra_javascript:
+  - "extra_js/init_ask_ai_widget.js"
 
 extra:
   analytics:

docs/package-lock.json

@@ -0,0 +1,132 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "Apache 2",
+      "dependencies": {
+        "apache-arrow": "file:../node/node_modules/apache-arrow",
+        "vectordb": "file:../node"
+      },
+      "devDependencies": {
+        "@types/node": "^20.11.8",
+        "typescript": "^5.3.3"
+      }
+    },
+    "../node": {
+      "name": "vectordb",
+      "version": "0.4.6",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
+        "axios": "^1.4.0"
+      },
+      "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
+        "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
+        "@types/mocha": "^10.0.1",
+        "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
+        "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
+        "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "cargo-cp-artifact": "^0.1",
+        "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
+        "eslint": "^8.39.0",
+        "eslint-config-standard-with-typescript": "^34.0.1",
+        "eslint-plugin-import": "^2.26.0",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1",
+        "mocha": "^10.2.0",
+        "openai": "^4.24.1",
+        "sinon": "^15.1.0",
+        "temp": "^0.9.4",
+        "ts-node": "^10.9.1",
+        "ts-node-dev": "^2.0.0",
+        "typedoc": "^0.24.7",
+        "typedoc-plugin-markdown": "^3.15.3",
+        "typescript": "*",
+        "uuid": "^9.0.0"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+      }
+    },
+    "../node/node_modules/apache-arrow": {
+      "version": "14.0.2",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "20.11.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
+      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "resolved": "../node/node_modules/apache-arrow",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
+      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
+    "node_modules/vectordb": {
+      "resolved": "../node",
+      "link": true
+    }
+  }
+}

docs/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "auto-generated tests from doc",
+  "author": "dev@lancedb.com",
+  "license": "Apache 2",
+  "dependencies": {
+    "apache-arrow": "file:../node/node_modules/apache-arrow",
+    "vectordb": "file:../node"
+  },
+  "scripts": {
+    "build": "tsc -b && cd ../node && npm run build-release",
+    "example": "npm run build && node",
+    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.8",
+    "typescript": "^5.3.3"
+  }
+}

docs/requirements.txt

@@ -1,5 +1,5 @@
-mkdocs==1.4.2
+mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
-mkdocs-material==9.1.3
+mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
 pydantic

docs/src/ann_indexes.md

@@ -1,29 +1,33 @@
-# ANN (Approximate Nearest Neighbor) Indexes
+# Approximate Nearest Neighbor (ANN) Indexes
 
-You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
+An ANN or a vector index is a data structure specifically designed to efficiently organize and
+search vector data based on their similarity via the chosen distance metric.
+By constructing a vector index, the search space is effectively narrowed down, avoiding the need
+for brute-force scanning of the entire vector space.
+A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-Currently, LanceDB does *not* automatically create the ANN index.
-LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+Currently, LanceDB does _not_ automatically create the ANN index.
+LanceDB has optimized code for kNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
+If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
-In the future we will look to automatically create and configure the ANN index.
+In the future we will look to automatically create and configure the ANN index as data comes in.
 
 ## Types of Index
 
 Lance can support multiple index types, the most widely used one is `IVF_PQ`.
 
-* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-   and then use **Product Quantization** to compress vectors in each partition.
-* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-   represent the nearest neighbors of each vector.
+- `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+  and then use **Product Quantization** to compress vectors in each partition.
+- `DiskANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+  represent the nearest neighbors of each vector.
 
 ## Creating an IVF_PQ Index
 
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
+
      Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
      ```python
@@ -43,25 +47,20 @@ Lance supports `IVF_PQ` index type by default.
      tbl.create_index(num_partitions=256, num_sub_vectors=96)
      ```
 
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
+=== "Typescript"
 
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('my_vectors', data)
-     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     ```typescript
+     --8<--- "docs/src/ann_indexes.ts:import"
+
+     --8<-- "docs/src/ann_indexes.ts:ingest"
      ```
 
 - **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
-We also support "cosine" and "dot" distance as well.
+  We also support "cosine" and "dot" distance as well.
 - **num_partitions** (default: 256): The number of partitions of the index.
 - **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-a single PQ code.
+  For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
+  a single PQ code.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -75,7 +74,7 @@ Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being
 
 You can specify the GPU device to train IVF partitions via
 
-- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
 
 === "Linux"
 
@@ -103,10 +102,9 @@ You can specify the GPU device to train IVF partitions via
 
 Trouble shootings:
 
-If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
 PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
 
-
 ## Querying an ANN Index
 
 Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
@@ -124,6 +122,7 @@ There are a couple of parameters that can be used to fine-tune the search:
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
+
      ```python
      tbl.search(np.random.random((1536))) \
          .limit(2) \
@@ -131,40 +130,35 @@ There are a couple of parameters that can be used to fine-tune the search:
          .refine_factor(10) \
          .to_pandas()
      ```
-     ```
+
+     ```text
                                               vector       item       _distance
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
      ```
 
-=== "Javascript"
-     ```javascript
-     const results_1 = await table
-         .search(Array(1536).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
+=== "Typescript"
+
+     ```typescript
+     --8<-- "docs/src/ann_indexes.ts:search1"
      ```
 
 The search will return the data requested in addition to the distance of each item.
 
-
 ### Filtering (where clause)
 
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
+
      ```python
      tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
-=== "Javascript"
+=== "Typescript"
+
      ```javascript
-     const results_2 = await table
-         .search(Array(1536).fill(1.2))
-         .where("id != '1141'")
-         .execute()
+     --8<-- "docs/src/ann_indexes.ts:search2"
      ```
 
 ### Projections (select clause)
@@ -172,22 +166,23 @@ You can further filter the elements returned by a search using a where clause.
 You can select the columns returned by the query using a select clause.
 
 === "Python"
+
      ```python
      tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
-     ```
-        vector                                             _distance
+
+
+     ```text
+                                                   vector _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
      ```
 
-=== "Javascript"
-     ```javascript
-     const results_3 = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .execute()
+=== "Typescript"
+
+     ```typescript
+     --8<-- "docs/src/ann_indexes.ts:search3"
      ```
 
 ## FAQ
@@ -216,4 +211,4 @@ On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows
 `num_sub_vectors` specifies how many Product Quantization (PQ) short codes to generate on each vector. Because
 PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in
 less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and
-more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.
+more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/ann_indexes.ts

@@ -0,0 +1,53 @@
+// --8<-- [start:import]
+import * as vectordb from "vectordb";
+// --8<-- [end:import]
+
+(async () => {
+  // --8<-- [start:ingest]
+  const db = await vectordb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    });
+  }
+  const table = await db.createTable("my_vectors", data);
+  await table.createIndex({
+    type: "ivf_pq",
+    column: "vector",
+    num_partitions: 16,
+    num_sub_vectors: 48,
+  });
+  // --8<-- [end:ingest]
+
+  // --8<-- [start:search1]
+  const results_1 = await table
+    .search(Array(1536).fill(1.2))
+    .limit(2)
+    .nprobes(20)
+    .refineFactor(10)
+    .execute();
+  // --8<-- [end:search1]
+
+  // --8<-- [start:search2]
+  const results_2 = await table
+    .search(Array(1536).fill(1.2))
+    .where("id != '1141'")
+    .limit(2)
+    .execute();
+  // --8<-- [end:search2]
+
+  // --8<-- [start:search3]
+  const results_3 = await table
+    .search(Array(1536).fill(1.2))
+    .select(["id"])
+    .limit(2)
+    .execute();
+  // --8<-- [end:search3]
+
+  console.log("Ann indexes: done");
+})();

docs/src/basic.md

@@ -1,91 +1,129 @@
-# Basic LanceDB Functionality
+# Quick start
 
-We'll cover the basics of using LanceDB on your local machine in this section.
+!!! info "LanceDB can be run in a number of ways:"
 
-??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
+    * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
+    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
+    * Deployed as a remote serverless database
 
-      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
+![](assets/lancedb_embedded_explanation.png)
 
 ## Installation
 
 === "Python"
+
       ```shell
       pip install lancedb
       ```
 
-=== "Javascript"
+=== "Typescript"
+
       ```shell
       npm install vectordb
       ```
 
+=== "Rust"
+
+    !!! warning "Rust SDK is experimental, might introduce breaking changes in the near future"
+
+    ```shell
+    cargo add vectordb
+    ```
+
+    !!! info "To use the vectordb create, you first need to install protobuf."
+
+    === "macOS"
+
+        ```shell
+        brew install protobuf
+        ```
+
+    === "Ubuntu/Debian"
+
+        ```shell
+        sudo apt install -y protobuf-compiler libssl-dev
+        ```
+
+    !!! info "Please also make sure you're using the same version of Arrow as in the [vectordb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+
 ## How to connect to a database
 
 === "Python"
+
       ```python
       import lancedb
       uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       ```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+=== "Typescript"
 
-      If you need a reminder of the uri, use the `db.uri` property.
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:import"
 
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
+    --8<-- "docs/src/basic_legacy.ts:open_db"
+    ```
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      ```
-      
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+=== "Rust"
 
-      If you need a reminder of the uri, you can call `db.uri()`.
+    ```rust
+    #[tokio::main]
+    async fn main() -> Result<()> {
+        --8<-- "rust/vectordb/examples/simple.rs:connect"
+    }
+    ```
+
+    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/vectordb/examples/simple.rs) for a full working example."
+
+LanceDB will create the directory if it doesn't exist (including parent directories).
+
+If you need a reminder of the uri, you can call `db.uri()`.
 
 ## How to create a table
 
 === "Python"
-      ```python
-      tbl = db.create_table("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `create_table` method.
+    ```python
+    tbl = db.create_table("my_table",
+                    data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                          {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    ```
 
-      You can also pass in a pandas DataFrame directly:
-      ```python
-      import pandas as pd
-      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      tbl = db.create_table("table_from_df", data=df)
-      ```
+    If the table already exists, LanceDB will raise an error by default.
+    If you want to overwrite the table, you can pass in `mode="overwrite"`
+    to the `create_table` method.
 
-      !!! warning
+    You can also pass in a pandas DataFrame directly:
 
-            If the table already exists, LanceDB will raise an error by default.
-            If you want to overwrite the table, you can pass in `mode="overwrite"`
-            to the `createTable` function.
+    ```python
+    import pandas as pd
+    df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    tbl = db.create_table("table_from_df", data=df)
+    ```
 
-=== "Javascript"
-      ```javascript
-      const tb = await db.createTable(
-        "myTable",
-        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
+=== "Typescript"
 
-      !!! warning
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_table"
+    ```
 
-            If the table already exists, LanceDB will raise an error by default.
-            If you want to overwrite the table, you can pass in `"overwrite"`
-            to the `createTable` function like this: `await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })`
-      
+    If the table already exists, LanceDB will raise an error by default.
+    If you want to overwrite the table, you can pass in `mode="overwrite"`
+    to the `createTable` function.
 
-??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+=== "Rust"
+
+    ```rust
+    use arrow_schema::{DataType, Schema, Field};
+    use arrow_array::{RecordBatch, RecordBatchIterator};
+
+    --8<-- "rust/vectordb/examples/simple.rs:create_table"
+    ```
+
+    If the table already exists, LanceDB will raise an error by default.
+
+!!! info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
 ### Creating an empty table
 
@@ -93,76 +131,145 @@ Sometimes you may not have the data to insert into the table at creation time.
 In this case, you can create an empty table and specify the schema.
 
 === "Python"
+
       ```python
       import pyarrow as pa
       schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
       tbl = db.create_table("empty_table", schema=schema)
       ```
 
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:create_empty_table"
+    ```
+
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
 
 === "Python"
-      ```python
-      tbl = db.open_table("my_table")
-      ```
 
-      If you forget the name of your table, you can always get a listing of all table names:
+    ```python
+    tbl = db.open_table("my_table")
+    ```
 
-      ```python
-      print(db.table_names())
-      ```
+=== "Typescript"
+
+    ```typescript
+    const tbl = await db.openTable("myTable");
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:open_with_existing_file"
+    ```
+
+If you forget the name of your table, you can always get a listing of all table names:
+
+=== "Python"
+
+    ```python
+    print(db.table_names())
+    ```
 
 === "Javascript"
-      ```javascript
-      const tbl = await db.openTable("myTable");
-      ```
 
-      If you forget the name of your table, you can always get a listing of all table names:
+    ```javascript
+    console.log(await db.tableNames());
+    ```
 
-      ```javascript
-      console.log(await db.tableNames());
-      ```
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:list_names"
+    ```
 
 ## How to add data to a table
 
 After a table has been created, you can always add more data to it using
 
 === "Python"
-      ```python
 
-      # Option 1: Add a list of dicts to a table
-      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+    ```python
+
+    # Option 1: Add a list of dicts to a table
+    data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
             {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
-      tbl.add(data)
+    tbl.add(data)
 
-      # Option 2: Add a pandas DataFrame to a table
-      df = pd.DataFrame(data)
-      tbl.add(data)
-      ```
+    # Option 2: Add a pandas DataFrame to a table
+    df = pd.DataFrame(data)
+    tbl.add(data)
+    ```
 
-=== "Javascript"
-      ```javascript
-      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:add"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:add"
+    ```
 
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:
 
 === "Python"
-      ```python
-      tbl.search([100, 100]).limit(2).to_pandas()
-      ```
 
-      This returns a pandas DataFrame with the results.
+    ```python
+    tbl.search([100, 100]).limit(2).to_pandas()
+    ```
 
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
-      ```
+    This returns a pandas DataFrame with the results.
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:search"
+    ```
+
+=== "Rust"
+
+    ```rust
+    use futures::TryStreamExt;
+
+    --8<-- "rust/vectordb/examples/simple.rs:search"
+    ```
+
+By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
+For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
+
+=== "Python"
+
+    ```py
+    tbl.create_index()
+    ```
+
+=== "Typescript"
+
+    ```{.typescript .ignore}
+    --8<-- "docs/src/basic_legacy.ts:create_index"
+    ```
+
+=== "Rust"
+
+    ```rust
+     --8<-- "rust/vectordb/examples/simple.rs:create_index"
+    ```
+
+Check [Approximate Nearest Neighbor (ANN) Indexes](/ann_indices.md) section for more details.
 
 ## How to delete rows from a table
 
@@ -171,20 +278,27 @@ which rows to delete, provide a filter that matches on the metadata columns.
 This can delete any number of rows that match the filter.
 
 === "Python"
-      ```python
-      tbl.delete('item = "fizz"')
-      ```
 
-=== "Javascript"
-      ```javascript
-      await tbl.delete('item = "fizz"')
-      ```
+    ```python
+    tbl.delete('item = "fizz"')
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:delete"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:delete"
+    ```
 
 The deletion predicate is a SQL expression that supports the same expressions
 as the `where()` clause on a search. They can be as simple or complex as needed.
 To see what expressions are supported, see the [SQL filters](sql.md) section.
 
-
 === "Python"
 
       Read more: [lancedb.table.Table.delete][]
@@ -198,6 +312,7 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
+
       ```python
       db.drop_table("my_table")
       ```
@@ -206,28 +321,37 @@ Use the `drop_table()` method on the database to remove a table.
       By default, if the table does not exist an exception is raised. To suppress this,
       you can pass in `ignore_missing=True`.
 
-=== "JavaScript"
-      ```javascript
-      await db.dropTable('myTable')
+=== "Typescript"
+
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
-      If the table does not exist an exception is raised. 
+      If the table does not exist an exception is raised.
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/vectordb/examples/simple.rs:drop_table"
+    ```
+
+!!! note "Bundling `vectordb` apps with Webpack"
+
+    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+    ```javascript
+    /** @type {import('next').NextConfig} */
+    module.exports = ({
+    webpack(config) {
+        config.externals.push({ vectordb: 'vectordb' })
+        return config;
+    }
+    })
+    ```
 
 ## What's next
 
-This section covered the very basics of the LanceDB API.
-LanceDB supports many additional features when creating indices to speed up search and options for search.
-These are contained in the next section of the documentation.
+This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
 
-## Note: Bundling vectorDB apps with webpack
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
-```javascript
-/** @type {import('next').NextConfig} */
-module.exports = ({
-  webpack(config) {
-    config.externals.push({ vectordb: 'vectordb' })
-    return config;
-  }
-})
-```
+If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.

docs/src/basic_legacy.ts

@@ -0,0 +1,92 @@
+// --8<-- [start:import]
+import * as lancedb from "vectordb";
+import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
+// --8<-- [end:import]
+import * as fs from "fs";
+import { Table as ArrowTable, Utf8 } from "apache-arrow";
+
+const example = async () => {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  // --8<-- [start:open_db]
+  const lancedb = require("vectordb");
+  const uri = "data/sample-lancedb";
+  const db = await lancedb.connect(uri);
+  // --8<-- [end:open_db]
+
+  // --8<-- [start:create_table]
+  const tbl = await db.createTable(
+    "myTable",
+    [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ],
+    { writeMode: lancedb.WriteMode.Overwrite }
+  );
+  // --8<-- [end:create_table]
+
+  // --8<-- [start:add]
+  const newData = Array.from({ length: 500 }, (_, i) => ({
+    vector: [i, i + 1],
+    item: "fizz",
+    price: i * 0.1,
+  }));
+  await tbl.add(newData);
+  // --8<-- [end:add]
+
+  // --8<-- [start:create_index]
+  await tbl.createIndex({
+    type: "ivf_pq",
+    num_partitions: 2,
+    num_sub_vectors: 2,
+  });
+  // --8<-- [end:create_index]
+
+  // --8<-- [start:create_empty_table]
+  const schema = new Schema([
+    new Field("id", new Int32()),
+    new Field("name", new Utf8()),
+  ]);
+  const empty_tbl = await db.createTable({ name: "empty_table", schema });
+  // --8<-- [end:create_empty_table]
+
+  // --8<-- [start:create_f16_table]
+  const dim = 16
+  const total = 10
+  const f16_schema = new Schema([
+      new Field('id', new Int32()),
+      new Field(
+        'vector',
+        new FixedSizeList(dim, new Field('item', new Float16(), true)),
+        false
+      )
+    ])
+  const data = lancedb.makeArrowTable(
+      Array.from(Array(total), (_, i) => ({
+        id: i,
+        vector: Array.from(Array(dim), Math.random)
+      })),
+      { f16_schema }
+    )
+  const table = await db.createTable('f16_tbl', data)
+  // --8<-- [end:create_f16_table]
+
+  // --8<-- [start:search]
+  const query = await tbl.search([100, 100]).limit(2).execute();
+  // --8<-- [end:search]
+  console.log(query);
+
+  // --8<-- [start:delete]
+  await tbl.delete('item = "fizz"');
+  // --8<-- [end:delete]
+
+  // --8<-- [start:drop_table]
+  await db.dropTable("myTable");
+  // --8<-- [end:drop_table]
+};
+
+async function main() {
+  await example();
+  console.log("Basic example: done");
+}
+
+main();

docs/src/cli_config.md

@@ -1,10 +1,15 @@
 
+# CLI & Config
+
 ## LanceDB CLI
-Once lanceDB is installed, you can access the CLI using `lancedb` command on the console
+Once lanceDB is installed, you can access the CLI using `lancedb` command on the console.
+
 ```
 lancedb
 ```
-This lists out all the various command-line options available. You can get the usage or help for a particular command
+
+This lists out all the various command-line options available. You can get the usage or help for a particular command.
+
 ```
 lancedb {command} --help
 ```
@@ -12,10 +17,13 @@ lancedb {command} --help
 ## LanceDB config
 LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
 To view your config settings, you can use:
+
 ```
 lancedb config
 ```
+
 These config parameters can be tuned using the cli.
+
 ```
 lancedb {config_name} --{argument}
 ```
@@ -23,15 +31,21 @@ lancedb {config_name} --{argument}
 ## LanceDB Opt-in Diagnostics
 When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
 These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
-Get usage help.
+
+### Get usage help
+
 ```
 lancedb diagnostics --help
 ```
-Disable diagnostics
+
+### Disable diagnostics
+
 ```
 lancedb diagnostics --disabled
 ```
-Enable diagnostics
+
+### Enable diagnostics
+
 ```
 lancedb diagnostics --enabled
-```
+```

docs/src/cloud/index.md

@@ -0,0 +1,17 @@
+# About LanceDB Cloud
+
+LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
+
+[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+
+## Architecture
+
+LanceDB Cloud provides the same underlying fast vector store that powers the OSS version, but without the need to maintain your own infrastructure. Because it's serverless, you only pay for the storage you use, and you can scale compute up and down as needed depending on the size of your data and its associated index.
+
+![](../assets/lancedb_cloud.png)
+
+## Transitioning from the OSS to the Cloud version
+
+The OSS version of LanceDB is designed to be embedded in your application, and it runs in-process. This makes it incredibly simple to self-host your own AI retrieval workflows for RAG and more and build and test out your concepts on your own infrastructure. The OSS version is forever free, and you can continue to build and integrate LanceDB into your existing backend applications without any added costs.
+
+Should you decide that you need a managed deployment in production, it's possible to seamlessly transition from the OSS to the cloud version by changing the connection string to point to a remote database instead of a local one. With LanceDB Cloud, you can take your AI application from development to production without major code changes or infrastructure burden.

docs/src/concepts/data_management.md

@@ -0,0 +1,62 @@
+# Data management
+
+This section covers concepts related to managing your data over time in LanceDB.
+
+## A primer on Lance
+
+Because LanceDB is built on top of the [Lance](https://lancedb.github.io/lance/) data format, it helps to understand some of its core ideas. Just like Apache Arrow, Lance is a fast columnar data format, but it has the added benefit of being versionable, query and train ML models on. Lance is designed to be used with simple and complex data types, like tabular data, images, videos audio, 3D point clouds (which are deeply nested) and more.
+
+The following concepts are important to keep in mind:
+
+- Data storage is columnar and is interoperable with other columnar formats (such as Parquet) via Arrow
+- Data is divided into fragments that represent a subset of the data
+- Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
+
+!!! note
+    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.  
+    2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
+
+## What are fragments?
+
+Fragments are chunks of data in a Lance dataset. Each fragment includes multiple files that contain several columns in the chunk of data that it represents.
+
+## Compaction
+
+As you insert more data, your dataset will grow and you'll need to perform *compaction* to maintain query throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments together to reduce the amount of metadata that needs to be managed, and to reduce the number of files that need to be opened while scanning the dataset.
+
+### How does compaction improve performance?
+
+Compaction performs the following tasks in the background:
+
+- Removes deleted rows from fragments
+- Removes dropped columns from fragments
+- Merges small fragments into larger ones
+
+Depending on the use case and dataset, optimal compaction will have different requirements. As a rule of thumb:
+
+- It’s always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
+- Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
+
+## Deletion
+
+Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
+
+## Reindexing
+
+Reindexing is the process of updating the index to account for new data, keeping good performance for queries. This applies to either a full-text search (FTS) index or a vector index. For ANN search, new data will always be included in query results, but queries on tables with unindexed data will fallback to slower search methods for the new parts of the table. This is another important operation to run periodically as your data grows, as it also improves performance. This is especially important if you're appending large amounts of data to an existing dataset.
+
+!!! tip
+    When adding new data to a dataset that has an existing index (either FTS or vector), LanceDB doesn't immediately update the index until a reindex operation is complete.
+
+Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
+
+When a reindex job is triggered in the background, the entire data is reindexed, but in the interim as new queries come in, LanceDB will combine results from the existing index with exhaustive kNN search on the new data. This is done to ensure that you're still searching on all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
+
+### Vector reindex
+
+* LanceDB Cloud supports incremental reindexing, where a background process will trigger a new index build for you automatically when new data is added to a dataset
+* LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well
+
+### FTS reindex
+
+FTS reindexing is supported in both LanceDB OSS and Cloud, but requires that it's manually rebuilt once you have a significant enough amount of new data added that needs to be reindexed. We [updated](https://github.com/lancedb/lancedb/pull/762) Tantivy's default heap size from 128MB to 1GB in LanceDB to make it much faster to reindex, by up to 10x from the default settings.

docs/src/concepts/index_ivfpq.md

@@ -0,0 +1,104 @@
+# Understanding LanceDB's IVF-PQ index
+
+An ANN (Approximate Nearest Neighbors) index is a data structure that represents data in a way that makes it more efficient to search and retrieve. Using an ANN index is faster, but less accurate than kNN or brute force search because, in essence, the index is a lossy representation of the data.
+
+LanceDB is fundamentally different from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy.
+
+## IVF-PQ
+
+IVF-PQ is a composite index that combines inverted file index (IVF) and product quantization (PQ). The implementation in LanceDB provides several parameters to fine-tune the index's size, query throughput, latency and recall, which are described later in this section.
+
+### Product quantization
+
+Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search.
+
+Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector.
+
+![](../assets/ivfpq_pq_desc.png)
+
+It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results.
+
+As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements.
+
+!!! example "Effect of quantization"
+
+    Original: `128 × 32 = 4096` bits
+    Quantized: `4 × 8 = 32` bits
+
+    Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial.
+
+### Inverted file index
+
+While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
+
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+
+![](../assets/ivfpq_ivf_desc.webp)
+
+During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query.
+
+![](../assets/ivfpq_query_vector.webp)
+
+## Putting it all together
+
+We can combine the above concepts to understand how to build and query an IVF-PQ index in LanceDB.
+
+### Construct index
+
+There are three key parameters to set when constructing an IVF-PQ index:
+
+* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `num_partitions`: The number of partitions in the IVF portion of the index.
+* `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
+
+In Python, the index can be created as follows:
+
+```python
+# Create and train the index for a 1536-dimensional vector
+# Make sure you have enough data in the table for an effective training step
+tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
+```
+
+The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
+
+
+### Query the index
+
+```python
+# Search using a random 1536-dimensional embedding
+tbl.search(np.random.random((1536))) \
+    .limit(2) \
+    .nprobes(20) \
+    .refine_factor(10) \
+    .to_pandas()
+```
+
+The above query will perform a search on the table `tbl` using the given query vector, with the following parameters:
+
+* `limit`: The number of results to return
+* `nprobes`: The number of probes determines the distribution of vector space. While a higher number enhances search accuracy, it also results in slower performance. Typically, setting `nprobes` to cover 5–10% of the dataset proves effective in achieving high recall with minimal latency.
+* `refine_factor`: Refine the results by reading extra elements and re-ranking them in memory. A higher number makes the search more accurate but also slower (see the [FAQ](../faq.md#do-i-need-to-set-a-refine-factor-when-using-an-index) page for more details on this).
+* `to_pandas()`: Convert the results to a pandas DataFrame
+
+And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
+
+
+## FAQ
+
+### When is it necessary to create a vector index?
+
+LanceDB has manually-tuned SIMD code for computing vector distances. In our benchmarks, computing 100K pairs of 1K dimension vectors takes **<20ms**. For small datasets (<100K rows) or applications that can accept up to 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how much memory will it take?
+
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with 1024-dimension vectors, if we choose `num_sub_vectors = 64`, each sub-vector has `1024 / 64 = 16` float32 numbers. Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for IVF_PQ index?
+
+`num_partitions` is used to decide how many partitions the first level IVF index uses. Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train. On SIFT-1M dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency/recall.
+
+`num_sub_vectors` specifies how many PQ short codes to generate on each vector. Because PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/concepts/storage.md

@@ -0,0 +1,80 @@
+# Storage
+
+LanceDB is among the only vector databases built on top of multiple modular components designed from the ground-up to be efficient on disk. This gives it the unique benefit of being flexible enough to support multiple storage backends, including local NVMe, EBS, EFS and many other third-party APIs that connect to the cloud.
+
+It is important to understand the tradeoffs between cost and latency for your specific application and use case. This section will help you understand the tradeoffs between the different storage backends.
+
+## Storage options
+
+We've prepared a simple diagram to showcase the thought process that goes into choosing a storage backend when using LanceDB OSS, Cloud or Enterprise.
+
+![](../assets/lancedb_storage_tradeoffs.png)
+
+When architecting your system, you'd typically ask yourself the following questions to decide on a storage option:
+
+1. **Latency**: How fast do I need results? What do the p50 and also p95 look like?
+2. **Scalability**: Can I scale up the amount of data and QPS easily?
+3. **Cost**: To serve my application, what’s the all-in cost of *both* storage and serving infra?
+4. **Reliability/Availability**: How does replication work? Is disaster recovery addressed?
+
+## Tradeoffs
+
+This section reviews the characteristics of each storage option in four dimensions: latency, scalability, cost and reliability.
+
+**We begin with the lowest cost option, and end with the lowest latency option.**
+
+### 1. S3 / GCS / Azure Blob Storage
+
+!!! tip "Lowest cost, highest latency"
+
+    - **Latency** ⇒ Has the highest latency. p95 latency is also substantially worse than p50. In general you get results in the order of several hundred milliseconds
+    - **Scalability** ⇒ Infinite on storage, however, QPS will be limited by S3 concurrency limits
+    - **Cost** ⇒ Lowest (order of magnitude cheaper than other options)
+    - **Reliability/Availability** ⇒ Highly available, as blob storage like S3 are critical infrastructure that form the backbone of the internet.
+
+Another important point to note is that LanceDB is designed to separate storage from compute, and the underlying Lance format stores the data in numerous immutable fragments. Due to these factors, LanceDB is a great storage option that addresses the _N + 1_ query problem. i.e., when a high query throughput is required, query processes can run in a stateless manner and be scaled up and down as needed.
+
+### 2. EFS / GCS Filestore / Azure File Storage
+
+!!! info "Moderately low cost, moderately low latency (<100ms)"
+
+    - **Latency** ⇒ Much better than object/blob storage but not as good as EBS/Local disk; < 100ms p95 achievable
+    - **Scalability** ⇒ High, but the bottleneck will be the IOPs limit, but when scaling you can provision multiple EFS volumes
+    - **Cost** ⇒ Significantly more expensive than S3 but still very cost effective compared to in-memory dbs. Inactive data in EFS is also automatically tiered to S3-level costs.
+    - **Reliability/Availability** ⇒ Highly available, as query nodes can go down without affecting EFS.  However, EFS does not provide replication / backup - this must be managed manually.
+
+A recommended best practice is to keep a copy of the data on S3 for disaster recovery scenarios. If any downtime is unacceptable, then you would need another EFS with a copy of the data. This is still much cheaper than EC2 instances holding multiple copies of the data.
+
+### 3. Third-party storage solutions
+
+Solutions like [MinIO](https://blog.min.io/lancedb-trusted-steed-against-data-complexity/), WekaFS, etc. that deliver S3 compatible API with much better performance than S3.
+
+!!! info "Moderately low cost, moderately low latency (<100ms)"
+
+    - **Latency** ⇒ Should be similar latency to EFS, better than S3 (<100ms)
+    - **Scalability** ⇒ Up to the solutions architect, who can add as many nodes to their MinIO or other third-party provider's cluster as needed
+    - **Cost** ⇒ Definitely higher than S3. The cost can be marginally higher than EFS until you get to maybe >10TB scale with high utilization
+    - **Reliability/Availability** ⇒ These are all shareable by lots of nodes, quality/cost of replication/backup depends on the vendor
+
+
+### 4. EBS / GCP Persistent Disk / Azure Managed Disk
+
+!!! info "Very low latency (<30ms), higher cost"
+
+    - **Latency** ⇒ Very good, pretty close to local disk. You’re looking at <30ms latency in most cases
+    - **Scalability** ⇒ EBS is not shareable between instances. If deployed via k8s, it can be shared between pods that live on the same instance, but beyond that you would need to shard data or make an additional copy
+    - **Cost** ⇒ Higher than EFS. There are some hidden costs to EBS as well if you’re paying for IO.
+    - **Reliability/Availability** ⇒ Not shareable between instances but can be shared between pods on the same instance. Survives instance termination. No automatic backups.
+
+Just like EFS, an EBS or persistent disk setup requires more manual work to manage data sharding, backups and capacity.
+
+### 5. Local disk (SSD/NVMe)
+
+!!! danger "Lowest latency (<10ms), highest cost"
+
+    - **Latency** ⇒ Lowest latency with modern NVMe drives, <10ms p95
+    - **Scalability** ⇒ Difficult to scale on cloud. Also need additional copies / sharding if QPS needs to be higher
+    - **Cost** ⇒ Highest cost; the main issue with keeping your application and storage tightly integrated is that it’s just not really possible to scale this up in cloud environments
+    - **Reliability/Availability** ⇒ If the instance goes down, so does your data. You have to be _very_ diligent about backing up your data
+
+As a rule of thumb, local disk should be your storage option if you require absolutely *crazy low* latency and you’re willing to do a bunch of data management work to make it happen.

docs/src/concepts/vector_search.md

@@ -0,0 +1,36 @@
+# Vector search
+
+Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search.
+
+Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a vector database like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups.
+
+![](../assets/vector-db-basics.png)
+
+## Embeddings
+
+Modern machine learning models can be trained to convert raw data into embeddings, represented as arrays (or vectors) of floating point numbers of fixed dimensionality. What makes embeddings useful in practice is that the position of an embedding in vector space captures some of the semantics of the data, depending on the type of model and how it was trained. Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar.
+
+Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model. Projecting the vectors' principal components in 2D space results in groups of vectors that represent similar concepts clustering together, as shown below.
+
+![](../assets/embedding_intro.png)
+
+## Indexes
+
+Embeddings for a given dataset are made searchable via an **index**. The index is constructed by using data structures that store the embeddings such that it's very efficient to perform scans and lookups on them. A key distinguishing feature of LanceDB is it uses a disk-based index: IVF-PQ, which is a variant of the Inverted File Index (IVF) that uses Product Quantization (PQ) to compress the embeddings.
+
+See the [IVF-PQ](./index_ivfpq.md) page for more details on how it works.
+
+## Brute force search
+
+The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. This is equivalent to a k-nearest neighbours (kNN) search in vector space.
+
+![](../assets/knn_search.png)
+
+As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in.
+
+## Approximate nearest neighbour (ANN) search
+
+Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases.
+
+
+

docs/src/embeddings/api.md

@@ -1,4 +1,5 @@
-To use your own custom embedding function, you need to follow these 2 simple steps.
+To use your own custom embedding function, you can follow these 2 simple steps:
+
 1. Create your embedding function by implementing the `EmbeddingFunction` interface
 2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
 
@@ -6,13 +7,11 @@ Let us see how this looks like in action.
 
 ![](../assets/embeddings_api.png)
 
+`EmbeddingFunction` and `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embedding function, you don't have to worry about the finer details - simply focus on setting up the model and leave the rest to LanceDB.
 
-`EmbeddingFunction` & `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embdding function, you don't need to worry about those details and simply focus on setting up the model.
-
-## `TextEmbeddingFunction` Interface
-
-There is another optional layer of abstraction provided in form of `TextEmbeddingFunction`. You can use this if your model isn't multi-modal in nature and only operates on text. In such case both source and vector fields will have the same pathway for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
+## `TextEmbeddingFunction` interface
 
+There is another optional layer of abstraction available: `TextEmbeddingFunction`. You can use this abstraction if your model isn't multi-modal in nature and only needs to operate on text. In such cases, both the source and vector fields will have the same work for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
 
 Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
 
@@ -39,7 +38,6 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
     @cached(cache={}) 
     def _embedding_model(self):
         return sentence_transformers.SentenceTransformer(name)
-
 ```
 
 This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.

docs/src/embeddings/default_embedding_functions.md

@@ -1,18 +1,19 @@
-There are various Embedding functions available out of the box with LanceDB. We're working on supporting other popular embedding APIs.
+There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models.
 
-## Text Embedding Functions
-Here are the text embedding functions registered by default.
-Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
-Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
+## Text embedding functions
+Contains the text embedding functions registered by default.
 
-### Sentence Transformers
-Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
+* Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential backoff. 
+* Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
+
+### Sentence transformers
+Allows you to set parameters when registering a `sentence-transformers` object.
 
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
-| `name` | `str` | `"all-MiniLM-L6-v2"` | The name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model. |
+| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
+| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
+| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
 
 
 ```python
@@ -37,15 +38,14 @@ actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
 
-### OpenAIEmbeddings
-LanceDB has OpenAI embeddings function in the registry by default. It is registered as `openai` and here are the parameters that you can customize when creating the instances
+### OpenAI embeddings
+LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
 
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
 
 
-
 ```python
 db = lancedb.connect("/tmp/db")
 registry = EmbeddingFunctionRegistry.get_instance()
@@ -69,17 +69,18 @@ print(actual.text)
 ```
 
 ### Instructor Embeddings
-Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
 
-If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
 
-                          Represent the `domain` `text_type` for `task_objective`:
+!!! info
+    Represent the `domain` `text_type` for `task_objective`:
 
-* `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
-* `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
-* `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
 
-More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
+More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
 
 | Argument | Type | Default | Description |
 |---|---|---|---|
@@ -118,12 +119,92 @@ texts = [{"text": "Capitalism has been dominant in the Western world since the e
 tbl.add(texts)
 ```
 
+### Gemini Embeddings
+With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
+The Gemini Embedding Model API supports various task types:
+
+| Task Type               | Description                                                                                                                                                |
+|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
+| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
+| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
+| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
+| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
+
+
+Usage Example:
+
+```python
+import lancedb
+import pandas as pd
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+
+model = get_registry().get("gemini-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
+### AWS Bedrock Text Embedding Functions
+AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
+You can do so by using `awscli` and also add your session_token:
+```shell
+aws configure
+aws configure set aws_session_token "<your_session_token>"
+```
+to ensure that the credentials are set up correctly, you can run the following command:
+```shell
+aws sts get-caller-identity
+```
+
+Supported Embedding modelIDs are:
+* `amazon.titan-embed-text-v1`
+* `cohere.embed-english-v3`
+* `cohere.embed-multilingual-v3`
+
+Supported paramters (to be passed in `create` method) are:
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
+| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
+| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
+| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
+| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
+| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
+| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
+
+Usage Example:
+
+```python
+model = get_registry().get("bedrock-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("tmp_path")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
 ## Multi-modal embedding functions
 Multi-modal embedding functions allow you to query your table using both images and text.
 
-### OpenClipEmbeddings
-We support CLIP model embeddings using the open source alternative, open-clip which supports various customizations. It is registered as `open-clip` and supports the following customizations:
-
+### OpenClip embeddings
+We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
 
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
@@ -133,11 +214,10 @@ We support CLIP model embeddings using the open source alternative, open-clip wh
 | `batch_size` | `int` | `64` | The number of images to process in a batch. |
 | `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
 
-
 This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
 
-NOTE:
-LanceDB supports ingesting images directly from accessible links.
+!!! info
+    LanceDB supports ingesting images directly from accessible links.
 
 
 ```python
@@ -205,4 +285,4 @@ print(actual.label)
 
 ```
 
-If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue.
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_explicit.md

@@ -0,0 +1,141 @@
+In this workflow, you define your own embedding function and pass it as a callable to LanceDB, invoking it in your code to generate the embeddings. Let's look at some examples.
+
+### Hugging Face
+
+!!! note
+    Currently, the Hugging Face method is only supported in the Python SDK.
+
+=== "Python"
+    The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
+    library, which can be installed via pip.
+
+    ```bash
+    pip install sentence-transformers
+    ```
+
+    The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
+    for a given document.
+
+    ```python
+    from sentence_transformers import SentenceTransformer
+
+    name="paraphrase-albert-small-v2"
+    model = SentenceTransformer(name)
+
+    # used for both training and querying
+    def embed_func(batch):
+        return [model.encode(sentence) for sentence in batch]
+    ```
+
+### OpenAI
+
+Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
+
+=== "Python"
+      ```python
+        import openai
+        import os
+
+        # Configuring the environment variable OPENAI_API_KEY
+        if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+
+        # verify that the API key is working
+        assert len(openai.Model.list()["data"]) > 0
+
+        def embed_func(c):
+            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+            return [record["embedding"] for record in rs["data"]]
+      ```
+
+=== "JavaScript"
+      ```javascript
+        const lancedb = require("vectordb");
+
+        // You need to provide an OpenAI API key
+        const apiKey = "sk-..."
+        // The embedding function will create embeddings for the 'text' column
+        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+      ```
+
+## Applying an embedding function to data
+
+=== "Python"
+    Using an embedding function, you can apply it to raw data
+    to generate embeddings for each record.
+
+    Say you have a pandas DataFrame with a `text` column that you want embedded,
+    you can use the `with_embeddings` function to generate embeddings and add them to 
+    an existing table.
+
+    ```python
+     import pandas as pd
+     from lancedb.embeddings import with_embeddings
+
+     df = pd.DataFrame(
+        [
+            {"text": "pepperoni"},
+            {"text": "pineapple"}
+        ]
+    )
+     data = with_embeddings(embed_func, df)
+
+     # The output is used to create / append to a table
+     # db.create_table("my_table", data=data)
+    ```
+
+    If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+
+    By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+    using the `batch_size` parameter to `with_embeddings`.
+
+    LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+    API call is reliable.
+
+=== "JavaScript"
+    Using an embedding function, you can apply it to raw data
+    to generate embeddings for each record.
+
+    Simply pass the embedding function created above and LanceDB will use it to generate
+    embeddings for your data.
+
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
+
+    const table = await db.createTable("vectors", data, embedding)
+    ```
+
+## Querying using an embedding function
+
+!!! warning
+    At query time, you **must** use the same embedding function you used to vectorize your data.
+    If you use a different embedding function, the embeddings will not reside in the same vector
+    space and the results will be nonsensical.
+
+=== "Python"
+     ```python
+     query = "What's the best pizza topping?"
+     query_vector = embed_func([query])[0]
+     results = (
+        tbl.search(query_vector)
+        .limit(10)
+        .to_pandas()
+     )
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "JavaScript"
+     ```javascript
+      const results = await table
+        .search("What's the best pizza topping?")
+        .limit(10)
+        .execute()
+     ```
+
+     The above snippet returns an array of records with the top 10 nearest neighbors to the query.

docs/src/embeddings/embedding_functions.md

@@ -1,79 +1,87 @@
-Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions can themselves be thought of as key part of the data processing pipeline that each request has to be passed through. The assumption here is: after initial setup, these components and the underlying methodology are not expected to change for a particular project.
 
-This is main motivation behind our new embedding functions API, that allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and simply focus on the DB aspects of VectorDB.
+For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
+!!! warning
+    Using the implicit embeddings management approach means that you can forget about the manually passing around embedding
+    functions in your code, as long as you don't intend to change it at a later time. If your embedding function changes,
+    you'll have to re-configure your table with the new embedding function and regenerate the embeddings.
 
-You can simply follow these steps and forget about the details of your embedding functions as long as you don't intend to change it.
-
-### Step 1 - Define the embedding function
-We have some pre-defined embedding functions in the global registry with more coming soon. Here's let's an implementation of CLIP as example.
+## 1. Define the embedding function
+We have some pre-defined embedding functions in the global registry, with more coming soon. Here's let's an implementation of CLIP as example.
 ```
 registry = EmbeddingFunctionRegistry.get_instance()
 clip = registry.get("open-clip").create()
 
 ```
-You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses PyDantic Model which can be utilized to write complex schemas simply as we'll see next!
+You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
 
-### Step 2 - Define the Data Model or Schema
-Our embedding function from the previous section abstracts away all the details about the models and dimensions required to define the schema. You can simply set a feild as **source** or **vector** column. Here's how
+## 2. Define the data model or schema
+The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
 ```python
 class Pets(LanceModel):
     vector: Vector(clip.ndims) = clip.VectorField()
     image_uri: str = clip.SourceField()
-
 ```
-`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for `vector` column & `SourceField` tells that when adding data, automatically use the embedding function to encode `image_uri`.
 
+`VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-### Step 3 - Create LanceDB Table
-Now that we have chosen/defined our embedding function and the schema, we can create the table
+## 3. Create LanceDB table
+Now that we have chosen/defined our embedding function and the schema, we can create the table:
 
 ```python
 db = lancedb.connect("~/lancedb")
 table = db.create_table("pets", schema=Pets)
 
 ```
-That's it! We have ingested all the information needed to embed source and query inputs. We can now forget about the model and dimension details and start to build or VectorDB
 
-### Step 4 - Ingest lots of data and run vector search!
-Now you can just add the data and it'll be vectorized automatically
+That's it! We've provided all the information needed to embed the source and query inputs. We can now forget about the model and dimension details and start to build our VectorDB pipeline.
+
+## 4. Ingest lots of data and query your table
+Any new or incoming data can just be added and it'll be vectorized automatically.
 
 ```python
 table.add([{"image_uri": u} for u in uris])
 ```
 
-Our OpenCLIP query embedding function support querying via both text and images. 
+Our OpenCLIP query embedding function supports querying via both text and images:
 
 ```python
 result = table.search("dog")
 ```
 
-Let's query an image
+Let's query an image:
 
 ```python
 p = Path("path/to/images/samoyed_100.jpg")
 query_image = Image.open(p)
 table.search(query_image)
-
 ```
-### Rate limit Handling
-`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default the maximum retires is set to 7. You can tune it by setting it to a different number or disable it by setting it to 0.
-Example
-----
+
+---
+
+## Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default, the maximum retires is set to 7. You can tune it by setting it to a different number, or disable it by setting it to 0.
+
+An example of how to do this is shown below:
 
 ```python
 clip = registry.get("open-clip").create() # Defaults to 7 max retries
 clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
 clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
-````
+```
 
-NOTE:
-Embedding functions can also fail due to other errors that have nothing to do with rate limits. This is why the error is also logged.
+!!! note
+    Embedding functions can also fail due to other errors that have nothing to do with rate limits.
+    This is why the error is also logged.
+
+## Some fun with Pydantic
+
+LanceDB is integrated with Pydantic, which was used in the example above to define the schema in Python. It's also used behind the scenes by the embedding function API to ingest useful information as table metadata.
+
+You can also use the integration for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let's define a utility function to plot the image.
 
-### A little fun with PyDantic
-LanceDB is integrated with PyDantic. Infact we've used the integration in the above example to define the schema. It is also being used behing the scene by the embdding function API to ingest useful information as table metadata.
-You can also use it for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let us define a utility function to plot the image.
 ```python
 class Pets(LanceModel):
     vector: Vector(clip.ndims) = clip.VectorField()
@@ -83,7 +91,7 @@ class Pets(LanceModel):
     def image(self):
         return Image.open(self.image_uri)
 ```
-Now, you can covert your search results to pydantic model and use this property.
+Now, you can covert your search results to a Pydantic model and use this property.
 
 ```python
 rs = table.search(query_image).limit(3).to_pydantic(Pets)
@@ -92,4 +100,4 @@ rs[2].image
 
 ![](../assets/dog_clip_output.png)
 
-Now that you've the basic idea about LanceDB embedding function, let us now dive deeper into the API that you can use to implement your own embedding functions!
+Now that you have the basic idea about implicit management via embedding functions, let's dive deeper into a [custom API](./api.md) that you can use to implement your own embedding functions.

docs/src/embeddings/index.md

@@ -1,149 +1,8 @@
-# Embedding
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. This makes them a very powerful tool for machine learning practitioners. However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
-Embeddings are high dimensional floating-point vector representations of your data or query. Anything can be embedded using some embedding model or function. Position of embedding in a high dimensional vector space has semantic significance to a degree that depends on the type of modal and training. These embeddings when projected in a 2-D space generally group similar entities close-by forming groups.
+LanceDB supports 2 methods of vectorizing your raw data into embeddings.
 
-![](../assets/embedding_intro.png)
+1. **Explicit**: By manually calling LanceDB's `with_embedding` function to vectorize your data via an `embed_func` of your choice
+2. **Implicit**: Allow LanceDB to embed the data and queries in the background as they come in, by using the table's `EmbeddingRegistry` information
 
-# Creating an embedding function
-
-LanceDB supports 2 major ways of vectorizing your data, explicit and implicit.
-
-1. By manually embedding the data before ingesting in the table
-2. By automatically embedding the data and query as they come, by ingesting embedding function information in the table itself! Covered in [Next Section](embedding_functions.md)
-
-Whatever workflow you prefer, we have the tools to support you.
-## Explicit Vectorization
-
-In this workflow, you can create your embedding function and vectorize your data using lancedb's `with_embedding` function. Let's look at some examples.
-
-### HuggingFace example
-
-One popular free option would be to use the [sentence-transformers](https://www.sbert.net/) library from HuggingFace.
-You can install this using pip: `pip install sentence-transformers`.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-name="paraphrase-albert-small-v2"
-model = SentenceTransformer(name)
-
-# used for both training and querying
-def embed_func(batch):
-    return [model.encode(sentence) for sentence in batch]
-```
-
-Please note that currently HuggingFace is only supported in the Python SDK.
-
-### OpenAI example
-
-You can also use an external API like OpenAI to generate embeddings
-
-=== "Python"
-      ```python
-        import openai
-        import os
-
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
-
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
-
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "Javascript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
-
-## Applying an embedding function
-
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each row.
-
-    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/python/#lancedb.embeddings.with_embeddings)
-    function to generate embeddings and add create a combined pyarrow table:
-
-
-     ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
-
-     df = pd.DataFrame([{"text": "pepperoni"},
-                        {"text": "pineapple"}])
-     data = with_embeddings(embed_func, df)
-
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-     ```
-
-     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-     using the `batch_size` parameter to `with_embeddings`.
-
-     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-     API call is reliable.
-
-=== "Javascript"
-     Using an embedding function, you can apply it to raw data
-     to generate embeddings for each row.
-
-     You can just pass the embedding function created previously and LanceDB will automatically generate
-     embededings for your data.
-
-      ```javascript
-      const db = await lancedb.connect("data/sample-lancedb");
-      const data = [
-        { text: 'pepperoni'  },
-        { text: 'pineapple' }
-      ]
-
-      const table = await db.createTable('vectors', data, embedding)
-      ```
-
-
-## Searching with an embedding function
-
-At inference time, you also need the same embedding function to embed your query text.
-It's important that you use the same model / function otherwise the embedding vectors don't
-belong in the same latent space and your results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_pandas()
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "Javascript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the 10 closest vectors to the query.
-
-
-## Implicit vectorization / Ingesting embedding functions
-Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions themselves be thought of as a part of the processing pipeline that each request(input) has to be passed through. After initial setup these components are not expected to change for a particular project. 
-
-This is main motivation behind our new embedding functions API, that allow you simply set it up once and the table remembers it, effectively making the **embedding functions disappear in the background** so you don't have to worry about modelling and simply focus on the DB aspects of VectorDB.
-
-Learn more in the Next Section
+See the [explicit](embedding_explicit.md) and [implicit](embedding_functions.md) embedding sections for more details.

docs/src/examples/examples_js.md

@@ -1,7 +1,6 @@
-# Examples
+# Examples: JavaScript
 
-Here are some of the examples, projects and applications using vectordb native javascript library.
-Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+To help you get started, we provide some examples, projects and applications that use the LanceDB JavaScript API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
 
 | Example | Scripts  |
 |-------- | ------   |
@@ -10,10 +9,3 @@ Some examples are covered in detail in the next sections. You can find more on [
 | [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
 | [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
 | [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |
-
-## Projects & Applications
-
-| Project Name                                        | Description                                                                                                          | Screenshot                                |
-|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
-| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
-| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/examples_python.md

@@ -0,0 +1,17 @@
+# Examples: Python
+
+To help you get started, we provide some examples, projects and applications that use the LanceDB Python API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
+
+| Example | Interactive Envs | Scripts  |
+|-------- | ---------------- | ------   |
+| | | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
+| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
+| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
+| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
+| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
+| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
+| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
+

docs/src/examples/index.md

@@ -1,21 +1,13 @@
-# Examples
+# Example projects and recipes
 
-Here are some of the examples, projects and applications using LanceDB python library. Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+## Recipes and example code
 
-| Example | Interactive Envs | Scripts  |
-|-------- | ---------------- | ------   |
-| | | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
-| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
-| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
-| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
-| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
-| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
-| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
+LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
 
-## Projects & Applications powered by LanceDB
+* 🐍 [Python](examples_python.md) examples
+* 👾 [JavaScript](exampled_js.md) examples
+
+## Applications powered by LanceDB
 
 | Project Name                                        | Description                                                                                                          | Screenshot                                |
 |-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|

docs/src/examples/modal_langchain.py

@@ -79,7 +79,10 @@ def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+    )
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -0,0 +1,11 @@
+document.addEventListener("DOMContentLoaded", function () {
+    var script = document.createElement("script");
+    script.src = "https://widget.kapa.ai/kapa-widget.bundle.js";
+    script.setAttribute("data-website-id", "c5881fae-cec0-490b-b45e-d83d131d4f25");
+    script.setAttribute("data-project-name", "LanceDB");
+    script.setAttribute("data-project-color", "#000000");
+    script.setAttribute("data-project-logo", "https://avatars.githubusercontent.com/u/108903835?s=200&v=4");
+  script.setAttribute("data-modal-example-questions","Help me create an IVF_PQ index,How do I do an exhaustive search?,How do I create a LanceDB table?,Can I use my own embedding function?");
+    script.async = true;
+    document.head.appendChild(script);
+  });

docs/src/faq.md

@@ -0,0 +1,87 @@
+This section covers some common questions and issues that you may encounter when using LanceDB.
+
+### Is LanceDB open source?
+
+Yes, LanceDB is an open source vector database available under an Apache 2.0 license. We also have a serverless SaaS solution, LanceDB Cloud, available under a commercial license.
+
+### What is the difference between Lance and LanceDB?
+
+[Lance](https://github.com/lancedb/lance) is a modern columnar data format for AI, written in Rust 🦀. It’s perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more.
+
+LanceDB is the vector database that’s built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA Bots to recommender systems.
+
+### Why invent another data format instead of using Parquet?
+
+As we mention in our talk titled “[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)”, Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that’s able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://blog.lancedb.com/benchmarking-random-access-in-lance-ed690757a826) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI.
+
+### Why build in Rust? 🦀
+
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
+
+### What is the difference between LanceDB OSS and LanceDB Cloud?
+
+LanceDB OSS is an **embedded** (in-process) solution that can be used as the vector store of choice for your LLM and RAG applications. It can be embedded inside an existing application backend, or used in-process alongside existing ML and data engineering pipelines.
+
+LanceDB Cloud is a **serverless** solution — the database and data sit on the cloud and we manage the scalability of the application side via a remote client, without the need to manage any infrastructure.
+
+Both flavors of LanceDB benefit from the blazing fast Lance data format and are built on the same open source foundations.
+
+### What makes LanceDB different?
+
+LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance’s multi-modal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata.
+
+The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure.
+
+### How large of a dataset can LanceDB handle?
+
+LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data (hundreds of terabytes). We are currently working with customers who regularly perform operations on 200M+ vectors, and we’re fast approaching billion scale and beyond, which are well-handled by our disk-based indexes, without you having to break the bank.
+
+### Do I need to build an ANN index to run vector search?
+
+No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
+
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+
+### Does LanceDB support full-text search?
+
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
+
+### How can I speed up data inserts?
+
+It's highly recommend to perform bulk inserts via batches (for e.g., Pandas DataFrames or lists of dicts in Python) to speed up inserts for large datasets. Inserting records one at a time is slow and can result in suboptimal performance because each insert creates a new data fragment on disk. Batching inserts allows LanceDB to create larger fragments (and their associated manifests), which are more efficient to read and write.
+
+### Do I need to set a refine factor when using an index?
+
+Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead.
+
+For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution.
+
+### How can I improve IVF-PQ recall while keeping latency low?
+
+When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in our [blog post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a).
+
+![](assets/recall-vs-latency.webp)
+
+### How do I connect to MinIO?
+
+MinIO supports an S3 compatible API. In order to connect to a MinIO instance, you need to:
+
+- Set the envvar `AWS_ENDPOINT` to the URL of your MinIO API
+- Set the envvars `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your MinIO credential
+- Call `lancedb.connect("s3://minio_bucket_name")`
+
+### Where can I find benchmarks for LanceDB?
+
+Refer to this [post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a) for recent benchmarks.
+
+### How much data can LanceDB practically manage without effecting performance?
+
+We target good performance on ~10-50 billion rows and ~10-30 TB of data.
+
+### Does LanceDB support concurrent operations?
+
+LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the [storage layer](https://lancedb.github.io/lancedb/concepts/storage/) you've chosen scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit
+
+!!! info "Multiprocessing with LanceDB"
+
+    For multiprocessing you should probably not use ```fork``` as lance is multi-threaded internally and ```fork``` and multi-thread do not work well.[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)

docs/src/fts.md

@@ -1,26 +1,21 @@
-# [EXPERIMENTAL] Full text search
+# Full-text search
 
-LanceDB now provides experimental support for full text search.
-This is currently Python only. We plan to push the integration down to Rust in the future
-to make this available for JS as well.
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
+
+A hybrid search solution combining vector and full-text search is also on the way.
 
 ## Installation
 
-To use full text search, you must install the dependency `tantivy-py`:
+To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
 
-# tantivy 0.20.1
 ```sh
+# Say you want to use tantivy==0.20.1
 pip install tantivy==0.20.1
 ```
 
+## Example
 
-## Quickstart
-
-Assume:
-1. `table` is a LanceDB Table
-2. `text` is the name of the `Table` column that we want to index
-
-For example,
+Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search.
 
 ```python
 import lancedb
@@ -28,36 +23,41 @@ import lancedb
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy", "meta": "foo"},
-                  {"vector": [5.9, 26.5], "text": "Sam was a loyal puppy", "meta": "bar"},
-                  {"vector": [15.9, 6.5], "text": "There are several kittens playing"}])
-
+table = db.create_table(
+    "my_table",
+    data=[
+        {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+        {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+    ],
+)
 ```
 
-To create the index:
+## Create FTS index on single column
+
+The FTS index must be created before you can search via keywords.
 
 ```python
 table.create_fts_index("text")
 ```
 
-To search:
+To search an FTS index via keywords, LanceDB's `table.search` accepts a string as input:
 
 ```python
 table.search("puppy").limit(10).select(["text"]).to_list()
 ```
 
-Which returns a list of dictionaries:
+This returns the result as a list of dictionaries as follows.
 
 ```python
 [{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
 ```
 
-LanceDB automatically looks for an FTS index if the input is str.
+!!! note
+    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
-## Multiple text columns
+## Index multiple columns
 
-If you have multiple columns to index, pass them all as a list to `create_fts_index`:
+If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
 
 ```python
 table.create_fts_index(["text1", "text2"])
@@ -75,13 +75,41 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
+## Syntax
+
+For full-text search you can perform either a phrase query like "the old man and the sea", 
+or a structured search query like "(Old AND Man) AND Sea".
+Double quotes are used to disambiguate.
+
+For example: 
+
+If you intended "they could have been dogs OR cats" as a phrase query, this actually
+raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
+this avoids the syntax error. However, it is cumbersome to have to remember what will
+conflict with the query syntax. Instead, if you search using 
+`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
+checking inside the quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can 
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
+```
+
 ## Current limitations
 
 1. Currently we do not yet support incremental writes.
-   If you add data after fts index creation, it won't be reflected
+   If you add data after FTS index creation, it won't be reflected
    in search results until you do a full reindex.
 
-2. We currently only support local filesystem paths for the fts index. 
+2. We currently only support local filesystem paths for the FTS index. 
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
 

docs/src/guides/storage.md

@@ -0,0 +1,167 @@
+# Configuring cloud storage
+
+<!-- TODO: When we add documentation for how to configure other storage types
+           we can change the name to a more general "Configuring storage" -->
+
+When using LanceDB OSS, you can choose where to store your data. The tradeoffs between different storage options are discussed in the [storage concepts guide](../concepts/storage.md). This guide shows how to configure LanceDB to use different storage options.
+
+## Object Stores
+
+LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure Blob Store, and Google Cloud Storage. Which object store to use is determined by the URI scheme of the dataset path. `s3://` is used for AWS S3, `az://` is used for Azure Blob Storage, and `gs://` is used for Google Cloud Storage. These URIs are passed to the `connect` function:
+
+=== "Python"
+
+    AWS S3:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("s3://bucket/path")
+    ```
+
+    Google Cloud Storage:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("gs://bucket/path")
+    ```
+
+    Azure Blob Storage:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("az://bucket/path")
+    ```
+
+=== "JavaScript"
+
+    AWS S3:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path");
+    ```
+
+    Google Cloud Storage:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("gs://bucket/path");
+    ```
+
+    Azure Blob Storage:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("az://bucket/path");
+    ```
+
+In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided using environment variables. In general, these environment variables are the same as those used by the respective cloud SDKs. The sections below describe the environment variables that can be used to configure each object store.
+
+LanceDB OSS uses the [object-store](https://docs.rs/object_store/latest/object_store/) Rust crate for object store access. There are general environment variables that can be used to configure the object store, such as the request timeout and proxy configuration. See the [object_store ClientConfigKey](https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html) doc for available configuration options. The environment variables that can be set are the snake-cased versions of these variable names. For example, to set `ProxyUrl` use the environment variable `PROXY_URL`. (Don't let the Rust docs intimidate you! We link to them so you can see an up-to-date list of the available options.)
+
+
+### AWS S3
+
+To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables.
+
+Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
+
+You can see a full list of environment variables [here](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html#method.from_env).
+
+!!! tip "Automatic cleanup for failed writes"
+
+    LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
+
+    **[Configuring a bucket lifecycle configuration to delete incomplete multipart uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html)**
+
+#### AWS IAM Permissions
+
+If a bucket is private, then an IAM policy must be specified to allow access to it. For many development scenarios, using broad permissions such as a PowerUser account is more than sufficient for working with LanceDB. However, in many production scenarios, you may wish to have as narrow as possible permissions.
+
+For **read and write access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:PutObject",
+              "s3:GetObject",
+              "s3:DeleteObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+For **read-only access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+#### S3-compatible stores
+
+LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify two environment variables: `AWS_ENDPOINT` and `AWS_DEFAULT_REGION`. `AWS_ENDPOINT` should be the URL of the S3-compatible store, and `AWS_DEFAULT_REGION` should be the region to use.
+
+<!-- TODO: we should also document the use of S3 Express once we fully support it -->
+
+### Google Cloud Storage
+
+GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. There are several aliases for this environment variable, documented [here](https://docs.rs/object_store/latest/object_store/gcp/struct.GoogleCloudStorageBuilder.html#method.from_env).
+
+
+!!! info "HTTP/2 support"
+
+    By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
+
+### Azure Blob Storage
+
+Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME` and ``AZURE_STORAGE_ACCOUNT_KEY`` environment variables. The full list of environment variables that can be set are documented [here](https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html#method.from_env).
+
+
+<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables.md

@@ -1,21 +1,37 @@
- <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+<a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/tables_guide.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
 A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
 
-This guide will show how to create tables, insert data into them, and update the data. You can follow along on colab!
+This guide will show how to create tables, insert data into them, and update the data.  
+
 
 ## Creating a LanceDB Table
 
 === "Python"
-    ### LanceDB Connection
+    Initialize a LanceDB connection and create a table using one of the many methods listed below.
 
     ```python
     import lancedb
     db = lancedb.connect("./.lancedb")
     ```
 
+=== "Javascript"
+
+    Initialize a VectorDB connection and create a table using one of the many methods listed below.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const uri = "data/sample-lancedb";
+    const db = await lancedb.connect(uri);
+    ```
+
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
-    ### From list of tuples or dictionaries
+### From list of tuples or dictionaries
+
+=== "Python"
 
     ```python
     import lancedb
@@ -29,16 +45,46 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
-
     !!! info "Note"
-        If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you can pass in mode="overwrite" to the createTable function.
+        If the table already exists, LanceDB will raise an error by default. 
+
+        `create_table` supports an optional `exist_ok` parameter. When set to True
+        and the table exists, then it simply opens the existing table. The data you
+        passed in will NOT be appended to the table in that case.
+
+        ```python
+        db.create_table("name", data, exist_ok=True)
+        ```
+        
+        Sometimes you want to make sure that you start fresh. If you want to 
+        overwrite the table, you can pass in mode="overwrite" to the createTable function.
 
         ```python
         db.create_table("name", data, mode="overwrite")
         ```
 
+=== "Javascript"
+    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
 
-    ### From pandas DataFrame
+    ```javascript
+    const tb = await db.createTable("my_table", [{
+        "vector": [3.1, 4.1],
+        "item": "foo",
+        "price": 10.0
+    }, {
+        "vector": [5.9, 26.5],
+        "item": "bar",
+        "price": 20.0
+    }]);
+    ```
+    !!! info "Note"
+    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
+
+    ```javascript
+    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```
+
+    ### From a Pandas DataFrame
 
     ```python
     import pandas as pd
@@ -49,12 +95,14 @@ This guide will show how to create tables, insert data into them, and update the
         "long": [-122.7, -74.1]
     })
 
-    db.create_table("table2", data)
+    db.create_table("my_table", data)
 
-    db["table2"].head()
+    db["my_table"].head()
     ```
     !!! info "Note"
-        Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+    Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+
+    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
 
     ```python
     custom_schema = pa.schema([
@@ -63,37 +111,73 @@ This guide will show how to create tables, insert data into them, and update the
     pa.field("long", pa.float32())
     ])
 
-    table = db.create_table("table3", data, schema=custom_schema)
+    table = db.create_table("my_table", data, schema=custom_schema)
     ```
 
-    ### From PyArrow Tables
-    You can also create LanceDB tables directly from pyarrow tables
+### From a Polars DataFrame
+
+    LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
+    written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
+    under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
+    is on the way.
 
     ```python
-    table = pa.Table.from_arrays(
-            [
-                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-                        pa.list_(pa.float32(), 4)),
-                pa.array(["foo", "bar"]),
-                pa.array([10.0, 20.0]),
-            ],
-            ["vector", "item", "price"],
-        )
+    import polars as pl
 
-    db = lancedb.connect("db")
+    data = pl.DataFrame({
+        "vector": [[3.1, 4.1], [5.9, 26.5]],
+        "item": ["foo", "bar"],
+        "price": [10.0, 20.0]
+    })
+    table = db.create_table("pl_table", data=data)
+    ```
 
-    tbl = db.create_table("test1", table)
+### From an Arrow Table
+=== "Python"
+    You can also create LanceDB tables directly from Arrow tables. 
+    LanceDB supports float16 data type!
+
+    ```python
+    import pyarrows as pa
+    import numpy as np
+    
+    dim = 16
+    total = 2
+    schema = pa.schema(
+        [
+            pa.field("vector", pa.list_(pa.float16(), dim)),
+            pa.field("text", pa.string())
+        ]
+    )
+    data = pa.Table.from_arrays(
+        [
+            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
+                    pa.list_(pa.float16(), dim)),
+            pa.array(["foo", "bar"])
+        ],
+        ["vector", "text"],
+    )
+    tbl = db.create_table("f16_tbl", data, schema=schema)
+    ```
+
+=== "Javascript"
+    You can also create LanceDB tables directly from Arrow tables. 
+    LanceDB supports Float16 data type!
+
+    ```javascript
+    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
     ```
 
     ### From Pydantic Models
+
     When you create an empty table without data, you must specify the table schema.
-    LanceDB supports creating tables by specifying a pyarrow schema or a specialized
-    pydantic model called `LanceModel`.
+    LanceDB supports creating tables by specifying a PyArrow schema or a specialized
+    Pydantic model called `LanceModel`.
 
     For example, the following Content model specifies a table with 5 columns:
-    movie_id, vector, genres, title, and imdb_id. When you create a table, you can
+    `movie_id`, `vector`, `genres`, `title`, and `imdb_id`. When you create a table, you can
     pass the class as the value of the `schema` parameter to `create_table`.
-    The `vector` column is a `Vector` type, which is a specialized pydantic type that
+    The `vector` column is a `Vector` type, which is a specialized Pydantic type that
     can be configured with the vector dimensions. It is also important to note that
     LanceDB only understands subclasses of `lancedb.pydantic.LanceModel`
     (which itself derives from `pydantic.BaseModel`).
@@ -157,8 +241,8 @@ This guide will show how to create tables, insert data into them, and update the
 
     #### Validators
 
-    Note that neither pydantic nor pyarrow automatically validates that input data
-    is of the *correct* timezone, but this is easy to add as a custom field validator:
+    Note that neither Pydantic nor PyArrow automatically validates that input data
+    is of the correct timezone, but this is easy to add as a custom field validator:
 
     ```python
     from datetime import datetime
@@ -198,9 +282,9 @@ This guide will show how to create tables, insert data into them, and update the
 
     ### Using Iterators / Writing Large Datasets
 
-    It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+    It is recommended to use iterators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
 
-    LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.
+    LanceDB additionally supports PyArrow's `RecordBatch` Iterators or other generators producing supported data types.
 
     Here's an example using using `RecordBatch` iterator for creating tables.
 
@@ -225,13 +309,47 @@ This guide will show how to create tables, insert data into them, and update the
         pa.field("price", pa.float32()),
     ])
 
-    db.create_table("table4", make_batches(), schema=schema)
+    db.create_table("batched_tale", make_batches(), schema=schema)
     ```
 
-    You can also use iterators of other types like Pandas dataframe or Pylists directly in the above example.
+    You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
-    ## Creating Empty Table
-    You can create empty tables in python. Initialize it with schema and later ingest data into it.
+## Open existing tables
+
+=== "Python"
+    If you forget the name of your table, you can always get a listing of all table names.
+
+    ```python
+    print(db.table_names())
+    ```
+
+    Then, you can open any existing tables.
+
+    ```python
+    tbl = db.open_table("my_table")
+    ```
+
+=== "JavaScript"
+    If you forget the name of your table, you can always get a listing of all table names.
+
+    ```javascript
+    console.log(await db.tableNames());
+    ```
+
+    Then, you can open any existing tables.
+
+    ```javascript
+    const tbl = await db.openTable("my_table");
+    ```
+
+## Creating empty table
+
+=== "Python"
+    In Python, you can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
+
+    ```python
+
+    An empty table can be initialized via a PyArrow schema.
 
     ```python
     import lancedb
@@ -243,132 +361,110 @@ This guide will show how to create tables, insert data into them, and update the
           pa.field("item", pa.string()),
           pa.field("price", pa.float32()),
       ])
-    tbl = db.create_table("table5", schema=schema)
-    data = [
-        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
-    ]
-    tbl.add(data=data)
+    tbl = db.create_table("empty_table_add", schema=schema)
     ```
 
-    You can also use Pydantic to specify the schema
+    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not 
+    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` 
+    that has been extended to support LanceDB specific types like `Vector`.
 
     ```python
     import lancedb
     from lancedb.pydantic import LanceModel, vector
 
-    class Model(LanceModel):
-          vector: Vector(2)
+    class Item(LanceModel):
+        vector: Vector(2)
+        item: str
+        price: float
 
-    tbl = db.create_table("table5", schema=Model.to_arrow_schema())
+    tbl = db.create_table("empty_table_add", schema=Item.to_arrow_schema())
     ```
 
-=== "Javascript/Typescript"
+    Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
-    ### VectorDB Connection
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
-    ### Creating a Table
-
-    You can create a LanceDB table in javascript using an array of records.
-
-    ```javascript
-    data
-    const tb = await db.createTable("my_table",
-      [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-    ```
-
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
-    ```
-
-## Open existing tables
-
-If you forget the name of your table, you can always get a listing of all table names:
+## Adding to a table
 
+After a table has been created, you can always add more data to it using the various methods available.
 
 === "Python"
-    ### Get a list of existing Tables
+    You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
 
-    ```python
-    print(db.table_names())
-    ```
-=== "Javascript/Typescript"
-
-    ```javascript
-    console.log(await db.tableNames());
-    ```
-
-Then, you can open any existing tables
-
-=== "Python"
-
-    ```python
-    tbl = db.open_table("my_table")
-    ```
-=== "Javascript/Typescript"
-
-    ```javascript
-    const tbl = await db.openTable("my_table");
-    ```
-
-## Adding to a Table
-After a table has been created, you can always add more data to it using
-
-=== "Python"
-    You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples.
-
-    ### Adding Pandas DataFrame
+    ### Add a Pandas DataFrame
 
     ```python
     df = pd.DataFrame({
-        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["fizz", "buzz"], "price": [100.0, 200.0]
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
     })
     tbl.add(df)
     ```
 
-    You can also add a large dataset batch in one go using Iterator of any supported data types.
+    ### Add a Polars DataFrame
 
-    ### Adding to table using Iterator
+    ```python
+    df = pl.DataFrame({
+        "vector": [[1.3, 1.4], [9.5, 56.2]], "item": ["banana", "apple"], "price": [5.0, 7.0]
+    })
+    tbl.add(df)
+    ```
+
+    ### Add an Iterator
+
+    You can also add a large dataset batch in one go using Iterator of any supported data types.
 
     ```python
     def make_batches():
         for i in range(5):
             yield [
-                    {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}
+                    {"vector": [3.1, 4.1], "item": "peach", "price": 6.0},
+                    {"vector": [5.9, 26.5], "item": "pear", "price": 5.0}
                 ]
     tbl.add(make_batches())
     ```
 
-    The other arguments accepted:
+    ### Add a PyArrow table
 
-    | Name | Type | Description | Default |
-    |---|---|---|---|
-    | data | DATA | The data to insert into the table. | required |
-    | mode | str | The mode to use when writing the data. Valid values are "append" and "overwrite". | append |
-    | on_bad_vectors | str | What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". | drop |
-    | fill value | float | The value to use when filling vectors: Only used if on_bad_vectors="fill". | 0.0 |
+    If you have data coming in as a PyArrow table, you can add it directly to the LanceDB table.
 
+    ```python
+    pa_table = pa.Table.from_arrays(
+            [
+                pa.array([[9.1, 6.7], [9.9, 31.2]],
+                        pa.list_(pa.float32(), 2)),
+                pa.array(["mango", "orange"]),
+                pa.array([7.0, 4.0]),
+            ],
+            ["vector", "item", "price"],
+        )
 
-=== "Javascript/Typescript"
-
-    ```javascript
-    await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-        {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
+    tbl.add(pa_table)
     ```
 
-## Deleting from a Table
+    ### Add a Pydantic Model
+
+    Assuming that a table has been created with the correct schema as shown [above](#creating-empty-table), you can add data items that are valid Pydantic models to the table.
+
+    ```python
+    pydantic_model_items = [
+        Item(vector=[8.1, 4.7], item="pineapple", price=10.0),
+        Item(vector=[6.9, 9.3], item="avocado", price=9.0)
+    ]
+
+    tbl.add(pydantic_model_items)
+    ```
+
+
+=== "JavaScript"
+
+    ```javascript
+    await tbl.add(
+        [
+            {vector: [1.3, 1.4], item: "fizz", price: 100.0},
+            {vector: [9.5, 56.2], item: "buzz", price: 200.0}
+        ]
+    )
+    ```
+
+## Deleting from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter.
 
@@ -413,7 +509,7 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     # 0  3  [5.0, 6.0]
     ```
 
-=== "Javascript/Typescript"
+=== "JavaScript"
 
     ```javascript
     await tbl.delete('item = "fizz"')
@@ -441,7 +537,7 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     await tbl.countRows() // Returns 1
     ```
 
-## Updating a Table
+## Updating a table
 
 This can be used to update zero to all rows depending on how many rows match the where clause. The update queries follow the form of a SQL UPDATE statement. The `where` parameter is a SQL filter that matches on the metadata columns. The `values` or `values_sql` parameters are used to provide the new values for the columns.
 
@@ -453,7 +549,7 @@ This can be used to update zero to all rows depending on how many rows match the
 
 !!! info "SQL syntax"
 
-    See [SQL filters](sql.md) for more information on the supported SQL syntax.
+    See [SQL filters](../sql.md) for more information on the supported SQL syntax.
 
 !!! warning "Warning"
 
@@ -492,9 +588,9 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
-=== "Javascript/Typescript"
+=== "JavaScript/Typescript"
 
-    API Reference: [vectordb.Table.update](../../javascript/interfaces/Table/#update)
+    API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
 
     ```javascript
     const lancedb = require("vectordb");
@@ -530,7 +626,7 @@ The `values` parameter is used to provide the new values for the columns as lite
     2  3  [10.0, 10.0]
     ```
 
-=== "Javascript/Typescript"
+=== "JavaScript/Typescript"
 
     ```javascript
     await tbl.update({ valuesSql: { x: "x + 1" } })
@@ -540,7 +636,6 @@ The `values` parameter is used to provide the new values for the columns as lite
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## What's next?
 
-## What's Next?
-
-Learn how to Query your tables and create indices
+Learn the best practices on creating an ANN index and getting the most out of it.

docs/src/hybrid_search.md

@@ -0,0 +1,235 @@
+# Hybrid Search
+
+LanceDB supports both semantic and keyword-based search. In real world applications, it is often useful to combine these two approaches to get the best best results. For example, you may want to search for a document that is semantically similar to a query document, but also contains a specific keyword. This is an example of *hybrid search*, a search algorithm that combines multiple search techniques.
+
+## Hybrid search in LanceDB
+You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
+
+```python
+import os
+
+import lancedb
+import openai
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+db = lancedb.connect("~/.lancedb")
+
+# Ingest embedding function in LanceDB table
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+    openai.api_key = "sk-..."
+embeddings = get_registry().get("openai").create()
+
+class Documents(LanceModel):
+    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
+    text: str = embeddings.SourceField()
+
+table = db.create_table("documents", schema=Documents)
+
+data = [
+    { "text": "rebel spaceships striking from a hidden base"},
+    { "text": "have won their first victory against the evil Galactic Empire"},
+    { "text": "during the battle rebel spies managed to steal secret plans"},
+    { "text": "to the Empire's ultimate weapon the Death Star"}
+]
+
+# ingest docs with auto-vectorization
+table.add(data)
+
+# Create a fts index before the hybrid search
+table.create_fts_index("text")
+# hybrid search with default re-ranker
+results = table.search("flower moon", query_type="hybrid").to_pandas()
+```
+
+By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
+
+
+### `rerank()` arguments
+* `normalize`: `str`, default `"score"`:
+    The method to normalize the scores. Can be "rank" or "score". If "rank", the scores are converted to ranks and then normalized. If "score", the scores are normalized directly.
+* `reranker`: `Reranker`, default `LinearCombinationReranker(weight=0.7)`.
+    The reranker to use. If not specified, the default reranker is used.
+
+
+## Available Rerankers
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
+
+### Linear Combination Reranker
+This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
+
+results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+Arguments
+----------------
+* `weight`: `float`, default `0.7`:
+    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
+* `fill`: `float`, default `1.0`:
+        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
+        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
+* `return_score` : str, default `"relevance"`
+        options are "relevance" or "all"
+        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
+
+### Cohere Reranker
+This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
+
+```python
+from lancedb.rerankers import CohereReranker
+
+reranker = CohereReranker()
+
+results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+Arguments
+----------------
+* `model_name`` : str, default `"rerank-english-v2.0"``
+        The name of the cross encoder model to use. Available cohere models are:
+        - rerank-english-v2.0
+        - rerank-multilingual-v2.0
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `top_n` : str, default `None`
+        The number of results to return. If None, will return all results.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### Cross Encoder Reranker
+This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
+
+```python
+from lancedb.rerankers import CrossEncoderReranker
+
+reranker = CrossEncoderReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+
+Arguments
+----------------
+* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
+        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `device` : str, default `None`
+        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+
+### ColBERT Reranker
+This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
+
+ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
+
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+Arguments
+----------------
+* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
+        The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### OpenAI Reranker
+This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
+
+!!! Note
+    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
+
+!!! Tip
+    You might run out of token limit so set the search `limits` based on your token limit.
+
+```python
+from lancedb.rerankers import OpenaiReranker
+
+reranker = OpenaiReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+Arguments
+----------------
+`model_name` : `str`, default `"gpt-3.5-turbo-1106"`
+    The name of the cross encoder model to use.
+`column` : `str`, default `"text"`
+    The name of the column to use as input to the cross encoder model.
+`return_score` : `str`, default `"relevance"`
+    options are "relevance" or "all". Only "relevance" is supported for now.
+`api_key` : `str`, default `None`
+    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+
+
+## Building Custom Rerankers
+You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
+
+The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    def __init__(self, param1, param2, ..., return_score="relevance"):
+        super().__init__(return_score)
+        self.param1 = param1
+        self.param2 = param2
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results
+        # ...
+
+        # Return the combined results
+        return combined_result
+
+```
+
+You can also accept additional arguments like a filter along with fts and vector search results
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    ...
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table, filter: str):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results & filter
+        # ...
+
+        # Return the combined results
+        return combined_result
+
+```

docs/src/index.md

@@ -1,75 +1,56 @@
 # LanceDB
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
+LanceDB is an open-source vector database for AI that's designed to store, manage, query and retrieve embeddings on large-scale multi-modal data. The core of LanceDB is written in Rust 🦀 and is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access.
 
-![Illustration](/lancedb/assets/ecosystem-illustration.png)
+Both the database and the underlying data format are designed from the ground up to be **easy-to-use**, **scalable** and **cost-effective**.
 
-The key features of LanceDB include:
+![](assets/lancedb_and_lance.png)
 
-* Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
+## Truly multi-modal
 
-* Support for production-scale vector similarity search, full-text search and SQL, with no servers to manage.
+Most existing vector databases that store and query just the embeddings and their metadata. The actual data is stored elsewhere, requiring you to manage their storage and versioning separately.
 
-* Native Python and Javascript/Typescript support.
+LanceDB supports storage of the *actual data itself*, alongside the embeddings and metadata. You can persist your images, videos, text documents, audio files and more in the Lance format, which provides automatic data versioning and blazing fast retrievals and filtering via LanceDB.
 
-* Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
+## Open-source and cloud solutions
 
-* Persisted on HDD, allowing scalability without breaking the bank.
+LanceDB is available in two flavors: **OSS** and **Cloud**.
 
-* Ingest your favorite data formats directly, like pandas DataFrames, Pydantic objects and more.
+LanceDB **OSS** is an **open-source**, batteries-included embedded vector database that you can run on your own infrastructure. "Embedded" means that it runs *in-process*, making it incredibly simple to self-host your own AI retrieval workflows for RAG and more. No servers, no hassle.
 
+LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverless in the cloud, making the storage clearly separated from compute. It's designed to be cost-effective and highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
 
-LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
+[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
 
-## Quick Start
+## Why use LanceDB?
 
-=== "Python"
-      ```shell
-      pip install lancedb
-      ```
+* Embedded (OSS) and serverless (Cloud) - no need to manage servers
 
-      ```python
-      import lancedb
+* Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
-      table = db.create_table("my_table",
-                              data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                                    {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      result = table.search([100, 100]).limit(2).to_list()
-      ```
+* Native Python and Javascript/Typescript support
 
-=== "Javascript"
-      ```shell
-      npm install vectordb
-      ```
+* Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
-      ```javascript
-      const lancedb = require("vectordb");
+* Tight integration with the [Arrow](https://arrow.apache.org/docs/format/Columnar.html) ecosystem, allowing true zero-copy access in shared memory with SIMD and GPU acceleration
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table",
-            [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
-            { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
-      const results = await table.search([100, 100]).limit(2).execute();
-      ```
+* Automatic data versioning to manage versions of your data without needing extra infrastructure
 
-## Complete Demos (Python)
-- [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
-- [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
-- [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
-- [Serverless QA Bot with S3 and Lambda](examples/serverless_lancedb_with_s3_and_lambda.md)
-- [Serverless QA Bot with Modal](examples/serverless_qa_bot_with_modal_and_langchain.md)
+* Disk-based index & storage, allowing for massive scalability without breaking the bank
 
-## Complete Demos (JavaScript)
-- [YouTube Transcript Search](examples/youtube_transcript_bot_with_nodejs.md)
+* Ingest your favorite data formats directly, like pandas DataFrames, Pydantic objects, Polars (coming soon), and more
 
-## Documentation Quick Links
-* [`Basic Operations`](basic.md) - basic functionality of LanceDB.
-* [`Embedding Functions`](embeddings/index.md) - functions for working with embeddings.
-* [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
-* [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
-* [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
-* [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Node SDK.
+## Documentation guide
+
+The following pages go deeper into the internal of LanceDB and how to use it.
+
+* [Quick start](basic.md): Get started with LanceDB and vector DB concepts
+* [Vector search concepts](concepts/vector_search.md): Understand the basics of vector search
+* [Working with tables](guides/tables.md): Learn how to work with tables and their associated functions
+* [Indexing](ann_indexes.md): Understand how to create indexes
+* [Vector search](search.md): Learn how to perform vector similarity search
+* [Full-text search](fts.md): Learn how to perform full-text search
+* [Managing embeddings](embeddings/index.md): Managing embeddings and the embedding functions API in LanceDB
+* [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
+* [Python API Reference](python/python.md): Python OSS and Cloud API references
+* [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references

docs/src/integrations/index.md

@@ -1,15 +1,13 @@
 # Integrations
 
-## Data Formats
+LanceDB supports ingesting from and exporting to your favorite data formats across the Python and JavaScript ecosystems.
 
-LanceDB supports ingesting from your favorite data tools.
-
-![Illustration](/lancedb/assets/ecosystem-illustration.png)
+![Illustration](../assets/ecosystem-illustration.png)
 
 
 ## Tools
 
-LanceDB is integrated with most of the popular AI tools, with more coming soon.
+LanceDB is integrated with a lot of popular AI tools, with more coming soon.
 Get started using these examples and quick links.
 
 | Integrations | |

docs/src/integrations/prompttools.md

@@ -1,7 +1,9 @@
 
 [PromptTools](https://github.com/hegelai/prompttools) offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.
+<!-- 
+[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> -->
 
-[Evaluating Prompts with PromptTools](./examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
+<a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
 
 ![Alt text](https://prompttools.readthedocs.io/en/latest/_images/demo.gif "a title")
 

docs/src/integrations/voxel51.md

@@ -1,31 +1,29 @@
-![example](/assets/voxel.gif)
+# FiftyOne
 
-Basic recipe
-____________
+FiftyOne is an open source toolkit for building high-quality datasets and computer vision models. It provides an API to create LanceDB tables and run similarity queries, both programmatically in Python and via point-and-click in the App.
 
-The basic workflow to use LanceDB to create a similarity index on your FiftyOne
-datasets and use this to query your data is as follows:
+![example](../assets/voxel.gif)
 
-1)  Load a dataset into FiftyOne
+## Basic recipe
 
-2)  Compute embedding vectors for samples or patches in your dataset, or select
-    a model to use to generate embeddings
+The basic workflow shown below uses LanceDB to create a similarity index on your FiftyOne
+datasets:
 
-3)  Use the `compute_similarity()`
-    method to generate a LanceDB table for the samples or object
-    patches embeddings in a dataset by setting the parameter `backend="lancedb"` and
-    specifying a `brain_key` of your choice
+1. Load a dataset into FiftyOne.
 
-4)  Use this LanceDB table to query your data with
-    `sort_by_similarity()`
+2. Compute embedding vectors for samples or patches in your dataset, or select a model to use to generate embeddings.
 
-5) If desired, delete the table
+3. Use the `compute_similarity()` method to generate a LanceDB table for the samples or object patches embeddings in a dataset by setting the parameter `backend="lancedb"` and specifying a `brain_key` of your choice.
+
+4. Use this LanceDB table to query your data with `sort_by_similarity()`.
+
+5. If desired, delete the table.
 
 The example below demonstrates this workflow.
 
 !!! Note
 
-    You must install the LanceDB Python client to run this
+    Install the LanceDB Python client to run the code shown below.
     ```
     pip install lancedb
     ```
@@ -68,4 +66,4 @@ lancedb_index.cleanup()
 dataset.delete_brain_run("lancedb_index")
 ```
 
-More in depth walkthrough of the integration, visit the LanceDB guide on Voxel51 - [LaceDB x Voxel51](https://docs.voxel51.com/integrations/lancedb.html)
+For a much more in depth walkthrough of the integration, visit the LanceDB x Voxel51 [docs page](https://docs.voxel51.com/integrations/lancedb.html).

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -38,4 +38,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation
 
 #### Defined in
 
-[index.ts:778](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L778)
+[index.ts:1019](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1019)

docs/src/javascript/classes/LocalConnection.md

@@ -46,7 +46,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L355)
+[index.ts:489](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L489)
 
 ## Properties
 
@@ -56,7 +56,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:353](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L353)
+[index.ts:487](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L487)
 
 ___
 
@@ -74,7 +74,7 @@ ___
 
 #### Defined in
 
-[index.ts:352](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L352)
+[index.ts:486](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L486)
 
 ## Accessors
 
@@ -92,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:360](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L360)
+[index.ts:494](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L494)
 
 ## Methods
 
@@ -127,7 +127,7 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Defined in
 
-[index.ts:395](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L395)
+[index.ts:542](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L542)
 
 ___
 
@@ -158,7 +158,7 @@ ___
 
 #### Defined in
 
-[index.ts:413](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L413)
+[index.ts:576](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L576)
 
 ___
 
@@ -184,7 +184,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:453](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L453)
+[index.ts:630](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L630)
 
 ___
 
@@ -210,7 +210,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:376](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L376)
+[index.ts:510](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L510)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -239,7 +239,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:384](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L384)
+[index.ts:518](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L518)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -266,7 +266,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:385](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L385)
+[index.ts:522](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L522)
 
 ___
 
@@ -286,4 +286,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:367](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L367)
+[index.ts:501](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L501)

docs/src/javascript/classes/LocalTable.md

@@ -23,6 +23,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Properties
 
 - [\_embeddings](LocalTable.md#_embeddings)
+- [\_isElectron](LocalTable.md#_iselectron)
 - [\_name](LocalTable.md#_name)
 - [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
@@ -31,16 +32,20 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 ### Accessors
 
 - [name](LocalTable.md#name)
+- [schema](LocalTable.md#schema)
 
 ### Methods
 
 - [add](LocalTable.md#add)
+- [checkElectron](LocalTable.md#checkelectron)
 - [cleanupOldVersions](LocalTable.md#cleanupoldversions)
 - [compactFiles](LocalTable.md#compactfiles)
 - [countRows](LocalTable.md#countrows)
 - [createIndex](LocalTable.md#createindex)
+- [createScalarIndex](LocalTable.md#createscalarindex)
 - [delete](LocalTable.md#delete)
 - [filter](LocalTable.md#filter)
+- [getSchema](LocalTable.md#getschema)
 - [indexStats](LocalTable.md#indexstats)
 - [listIndices](LocalTable.md#listindices)
 - [overwrite](LocalTable.md#overwrite)
@@ -69,7 +74,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L464)
+[index.ts:642](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L642)
 
 • **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
@@ -90,7 +95,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:471](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L471)
+[index.ts:649](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L649)
 
 ## Properties
 
@@ -100,7 +105,17 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:461](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L461)
+[index.ts:639](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L639)
+
+___
+
+### \_isElectron
+
+• `Private` `Readonly` **\_isElectron**: `boolean`
+
+#### Defined in
+
+[index.ts:638](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L638)
 
 ___
 
@@ -110,7 +125,7 @@ ___
 
 #### Defined in
 
-[index.ts:460](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L460)
+[index.ts:637](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L637)
 
 ___
 
@@ -128,7 +143,7 @@ ___
 
 #### Defined in
 
-[index.ts:462](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L462)
+[index.ts:640](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L640)
 
 ___
 
@@ -138,7 +153,7 @@ ___
 
 #### Defined in
 
-[index.ts:459](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L459)
+[index.ts:636](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L636)
 
 ___
 
@@ -164,7 +179,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:499](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L499)
+[index.ts:688](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L688)
 
 ## Accessors
 
@@ -182,7 +197,25 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:479](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L479)
+[index.ts:668](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L668)
+
+___
+
+### schema
+
+• `get` **schema**(): `Promise`\<`Schema`\<`any`\>\>
+
+#### Returns
+
+`Promise`\<`Schema`\<`any`\>\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[schema](../interfaces/Table.md#schema)
+
+#### Defined in
+
+[index.ts:849](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L849)
 
 ## Methods
 
@@ -210,7 +243,21 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:507](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L507)
+[index.ts:696](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L696)
+
+___
+
+### checkElectron
+
+▸ `Private` **checkElectron**(): `boolean`
+
+#### Returns
+
+`boolean`
+
+#### Defined in
+
+[index.ts:861](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L861)
 
 ___
 
@@ -233,7 +280,7 @@ Clean up old versions of the table, freeing disk space.
 
 #### Defined in
 
-[index.ts:596](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L596)
+[index.ts:808](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L808)
 
 ___
 
@@ -260,7 +307,7 @@ Metrics about the compaction operation.
 
 #### Defined in
 
-[index.ts:615](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L615)
+[index.ts:831](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L831)
 
 ___
 
@@ -280,7 +327,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:543](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L543)
+[index.ts:749](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L749)
 
 ___
 
@@ -310,7 +357,42 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:536](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L536)
+[index.ts:734](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L734)
+
+___
+
+### createScalarIndex
+
+▸ **createScalarIndex**(`column`, `replace`): `Promise`\<`void`\>
+
+Create a scalar index on this Table for the given column
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `column` | `string` | The column to index |
+| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect('././lancedb')
+const table = await con.openTable('images')
+await table.createScalarIndex('my_col')
+```
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createScalarIndex](../interfaces/Table.md#createscalarindex)
+
+#### Defined in
+
+[index.ts:742](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L742)
 
 ___
 
@@ -336,7 +418,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:552](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L552)
+[index.ts:758](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L758)
 
 ___
 
@@ -358,7 +440,21 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:495](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L495)
+[index.ts:684](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L684)
+
+___
+
+### getSchema
+
+▸ `Private` **getSchema**(): `Promise`\<`Schema`\<`any`\>\>
+
+#### Returns
+
+`Promise`\<`Schema`\<`any`\>\>
+
+#### Defined in
+
+[index.ts:854](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L854)
 
 ___
 
@@ -384,7 +480,7 @@ Get statistics about an index.
 
 #### Defined in
 
-[index.ts:628](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L628)
+[index.ts:845](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L845)
 
 ___
 
@@ -404,7 +500,7 @@ List the indicies on this table.
 
 #### Defined in
 
-[index.ts:624](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L624)
+[index.ts:841](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L841)
 
 ___
 
@@ -432,7 +528,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:522](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L522)
+[index.ts:716](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L716)
 
 ___
 
@@ -458,7 +554,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:487](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L487)
+[index.ts:676](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L676)
 
 ___
 
@@ -484,4 +580,4 @@ Update rows in this table.
 
 #### Defined in
 
-[index.ts:563](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L563)
+[index.ts:771](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L771)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -65,7 +65,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L38)
+[query.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L38)
 
 ## Properties
 
@@ -75,7 +75,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L36)
+[query.ts:36](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L36)
 
 ___
 
@@ -85,7 +85,7 @@ ___
 
 #### Defined in
 
-[query.ts:33](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L33)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L33)
 
 ___
 
@@ -95,7 +95,7 @@ ___
 
 #### Defined in
 
-[query.ts:29](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L29)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L29)
 
 ___
 
@@ -105,7 +105,7 @@ ___
 
 #### Defined in
 
-[query.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L34)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L34)
 
 ___
 
@@ -115,7 +115,7 @@ ___
 
 #### Defined in
 
-[query.ts:31](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L31)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L31)
 
 ___
 
@@ -125,7 +125,7 @@ ___
 
 #### Defined in
 
-[query.ts:35](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L35)
+[query.ts:35](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L35)
 
 ___
 
@@ -135,7 +135,7 @@ ___
 
 #### Defined in
 
-[query.ts:26](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L26)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L26)
 
 ___
 
@@ -145,7 +145,7 @@ ___
 
 #### Defined in
 
-[query.ts:28](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L28)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L28)
 
 ___
 
@@ -155,7 +155,7 @@ ___
 
 #### Defined in
 
-[query.ts:30](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L30)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L30)
 
 ___
 
@@ -165,7 +165,7 @@ ___
 
 #### Defined in
 
-[query.ts:32](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L32)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L32)
 
 ___
 
@@ -175,7 +175,7 @@ ___
 
 #### Defined in
 
-[query.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L27)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L27)
 
 ___
 
@@ -201,7 +201,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:87](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L87)
+[query.ts:87](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L87)
 
 ## Methods
 
@@ -223,7 +223,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[query.ts:115](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L115)
+[query.ts:115](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L115)
 
 ___
 
@@ -245,7 +245,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L82)
+[query.ts:82](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L82)
 
 ___
 
@@ -259,7 +259,7 @@ ___
 
 #### Defined in
 
-[query.ts:142](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L142)
+[query.ts:142](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L142)
 
 ___
 
@@ -281,7 +281,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[query.ts:55](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L55)
+[query.ts:55](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L55)
 
 ___
 
@@ -307,7 +307,7 @@ MetricType for the different options
 
 #### Defined in
 
-[query.ts:102](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L102)
+[query.ts:102](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L102)
 
 ___
 
@@ -329,7 +329,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[query.ts:73](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L73)
+[query.ts:73](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L73)
 
 ___
 
@@ -349,7 +349,7 @@ ___
 
 #### Defined in
 
-[query.ts:107](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L107)
+[query.ts:107](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L107)
 
 ___
 
@@ -371,7 +371,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[query.ts:64](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L64)
+[query.ts:64](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L64)
 
 ___
 
@@ -393,4 +393,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[query.ts:93](https://github.com/lancedb/lancedb/blob/7856a94/node/src/query.ts#L93)
+[query.ts:93](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L93)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:798](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L798)
+[index.ts:1041](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1041)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:803](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L803)
+[index.ts:1046](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1046)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:793](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L793)
+[index.ts:1036](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1036)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:766](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L766)
+[index.ts:1007](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1007)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:762](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L762)
+[index.ts:1003](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1003)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:764](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L764)
+[index.ts:1005](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1005)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L34)
+[index.ts:54](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L54)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:36](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L36)
+[index.ts:56](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L56)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:38](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L38)
+[index.ts:58](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L58)

docs/src/javascript/interfaces/CleanupStats.md

@@ -19,7 +19,7 @@ The number of bytes removed from disk.
 
 #### Defined in
 
-[index.ts:637](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L637)
+[index.ts:878](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L878)
 
 ___
 
@@ -31,4 +31,4 @@ The number of old table versions removed.
 
 #### Defined in
 
-[index.ts:641](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L641)
+[index.ts:882](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L882)

docs/src/javascript/interfaces/CompactionMetrics.md

@@ -22,7 +22,7 @@ fragments added.
 
 #### Defined in
 
-[index.ts:692](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L692)
+[index.ts:933](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L933)
 
 ___
 
@@ -35,7 +35,7 @@ file.
 
 #### Defined in
 
-[index.ts:687](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L687)
+[index.ts:928](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L928)
 
 ___
 
@@ -47,7 +47,7 @@ The number of new fragments that were created.
 
 #### Defined in
 
-[index.ts:682](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L682)
+[index.ts:923](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L923)
 
 ___
 
@@ -59,4 +59,4 @@ The number of fragments that were removed.
 
 #### Defined in
 
-[index.ts:678](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L678)
+[index.ts:919](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L919)

docs/src/javascript/interfaces/CompactionOptions.md

@@ -24,7 +24,7 @@ Default is true.
 
 #### Defined in
 
-[index.ts:660](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L660)
+[index.ts:901](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L901)
 
 ___
 
@@ -38,7 +38,7 @@ the deleted rows. Default is 10%.
 
 #### Defined in
 
-[index.ts:666](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L666)
+[index.ts:907](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L907)
 
 ___
 
@@ -50,7 +50,7 @@ The maximum number of rows per group. Defaults to 1024.
 
 #### Defined in
 
-[index.ts:654](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L654)
+[index.ts:895](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L895)
 
 ___
 
@@ -63,7 +63,7 @@ the number of cores on the machine.
 
 #### Defined in
 
-[index.ts:671](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L671)
+[index.ts:912](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L912)
 
 ___
 
@@ -77,4 +77,4 @@ Defaults to 1024 * 1024.
 
 #### Defined in
 
-[index.ts:650](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L650)
+[index.ts:891](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L891)

docs/src/javascript/interfaces/Connection.md

@@ -31,7 +31,7 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:125](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L125)
+[index.ts:183](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L183)
 
 ## Methods
 
@@ -59,7 +59,7 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Defined in
 
-[index.ts:146](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L146)
+[index.ts:207](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L207)
 
 ▸ **createTable**(`name`, `data`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
 
@@ -78,7 +78,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:154](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L154)
+[index.ts:221](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L221)
 
 ▸ **createTable**(`name`, `data`, `options`): `Promise`\<[`Table`](Table.md)\<`number`[]\>\>
 
@@ -98,7 +98,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:163](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L163)
+[index.ts:233](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L233)
 
 ▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
@@ -124,7 +124,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:172](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L172)
+[index.ts:246](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L246)
 
 ▸ **createTable**\<`T`\>(`name`, `data`, `embeddings`, `options`): `Promise`\<[`Table`](Table.md)\<`T`\>\>
 
@@ -151,7 +151,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:181](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L181)
+[index.ts:259](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L259)
 
 ___
 
@@ -173,7 +173,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:187](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L187)
+[index.ts:270](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L270)
 
 ___
 
@@ -202,7 +202,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:135](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L135)
+[index.ts:193](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L193)
 
 ___
 
@@ -216,4 +216,4 @@ ___
 
 #### Defined in
 
-[index.ts:127](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L127)
+[index.ts:185](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L185)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -21,7 +21,7 @@
 
 #### Defined in
 
-[index.ts:49](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L49)
+[index.ts:81](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L81)
 
 ___
 
@@ -29,9 +29,13 @@ ___
 
 • `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
 
+User provided AWS crednetials.
+
+If not provided, LanceDB will use the default credentials provider chain.
+
 #### Defined in
 
-[index.ts:44](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L44)
+[index.ts:75](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L75)
 
 ___
 
@@ -39,9 +43,11 @@ ___
 
 • `Optional` **awsRegion**: `string`
 
+AWS region to connect to. Default is defaultAwsRegion.
+
 #### Defined in
 
-[index.ts:46](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L46)
+[index.ts:78](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L78)
 
 ___
 
@@ -49,9 +55,13 @@ ___
 
 • `Optional` **hostOverride**: `string`
 
+Override the host URL for the remote connections.
+
+This is useful for local testing.
+
 #### Defined in
 
-[index.ts:54](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L54)
+[index.ts:91](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L91)
 
 ___
 
@@ -59,9 +69,11 @@ ___
 
 • `Optional` **region**: `string`
 
+Region to connect
+
 #### Defined in
 
-[index.ts:51](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L51)
+[index.ts:84](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L84)
 
 ___
 
@@ -69,6 +81,12 @@ ___
 
 • **uri**: `string`
 
+LanceDB database URI.
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (SaaS)
+
 #### Defined in
 
-[index.ts:42](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L42)
+[index.ts:69](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L69)

docs/src/javascript/interfaces/CreateTableOptions.md

@@ -26,7 +26,7 @@
 
 #### Defined in
 
-[index.ts:79](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L79)
+[index.ts:116](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L116)
 
 ___
 
@@ -36,7 +36,7 @@ ___
 
 #### Defined in
 
-[index.ts:85](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L85)
+[index.ts:122](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L122)
 
 ___
 
@@ -46,7 +46,7 @@ ___
 
 #### Defined in
 
-[index.ts:76](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L76)
+[index.ts:113](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L113)
 
 ___
 
@@ -56,7 +56,7 @@ ___
 
 #### Defined in
 
-[index.ts:82](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L82)
+[index.ts:119](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L119)
 
 ___
 
@@ -66,4 +66,4 @@ ___
 
 #### Defined in
 
-[index.ts:88](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L88)
+[index.ts:125](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L125)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7856a94/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IndexStats.md

@@ -17,7 +17,7 @@
 
 #### Defined in
 
-[index.ts:344](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L344)
+[index.ts:478](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L478)
 
 ___
 
@@ -27,4 +27,4 @@ ___
 
 #### Defined in
 
-[index.ts:345](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L345)
+[index.ts:479](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L479)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -29,7 +29,7 @@ The column to be indexed
 
 #### Defined in
 
-[index.ts:701](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L701)
+[index.ts:942](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L942)
 
 ___
 
@@ -41,7 +41,7 @@ Cache size of the index
 
 #### Defined in
 
-[index.ts:750](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L750)
+[index.ts:991](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L991)
 
 ___
 
@@ -53,7 +53,7 @@ A unique name for the index
 
 #### Defined in
 
-[index.ts:706](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L706)
+[index.ts:947](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L947)
 
 ___
 
@@ -65,7 +65,7 @@ The max number of iterations for kmeans training.
 
 #### Defined in
 
-[index.ts:721](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L721)
+[index.ts:962](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L962)
 
 ___
 
@@ -77,7 +77,7 @@ Max number of iterations to train OPQ, if `use_opq` is true.
 
 #### Defined in
 
-[index.ts:740](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L740)
+[index.ts:981](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L981)
 
 ___
 
@@ -89,7 +89,7 @@ Metric type, L2 or Cosine
 
 #### Defined in
 
-[index.ts:711](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L711)
+[index.ts:952](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L952)
 
 ___
 
@@ -101,7 +101,7 @@ The number of bits to present one PQ centroid.
 
 #### Defined in
 
-[index.ts:735](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L735)
+[index.ts:976](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L976)
 
 ___
 
@@ -113,7 +113,7 @@ The number of partitions this index
 
 #### Defined in
 
-[index.ts:716](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L716)
+[index.ts:957](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L957)
 
 ___
 
@@ -125,7 +125,7 @@ Number of subvectors to build PQ code
 
 #### Defined in
 
-[index.ts:731](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L731)
+[index.ts:972](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L972)
 
 ___
 
@@ -137,7 +137,7 @@ Replace an existing index with the same name if it exists.
 
 #### Defined in
 
-[index.ts:745](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L745)
+[index.ts:986](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L986)
 
 ___
 
@@ -147,7 +147,7 @@ ___
 
 #### Defined in
 
-[index.ts:752](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L752)
+[index.ts:993](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L993)
 
 ___
 
@@ -159,4 +159,4 @@ Train as optimized product quantization.
 
 #### Defined in
 
-[index.ts:726](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L726)
+[index.ts:967](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L967)

docs/src/javascript/interfaces/Table.md

@@ -21,11 +21,13 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 - [add](Table.md#add)
 - [countRows](Table.md#countrows)
 - [createIndex](Table.md#createindex)
+- [createScalarIndex](Table.md#createscalarindex)
 - [delete](Table.md#delete)
 - [indexStats](Table.md#indexstats)
 - [listIndices](Table.md#listindices)
 - [name](Table.md#name)
 - [overwrite](Table.md#overwrite)
+- [schema](Table.md#schema)
 - [search](Table.md#search)
 - [update](Table.md#update)
 
@@ -55,7 +57,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:209](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L209)
+[index.ts:291](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L291)
 
 ___
 
@@ -75,7 +77,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:229](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L229)
+[index.ts:361](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L361)
 
 ___
 
@@ -105,7 +107,42 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:224](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L224)
+[index.ts:306](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L306)
+
+___
+
+### createScalarIndex
+
+• **createScalarIndex**: (`column`: `string`, `replace`: `boolean`) => `Promise`\<`void`\>
+
+#### Type declaration
+
+▸ (`column`, `replace`): `Promise`\<`void`\>
+
+Create a scalar index on this Table for the given column
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `column` | `string` | The column to index |
+| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+
+##### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect('././lancedb')
+const table = await con.openTable('images')
+await table.createScalarIndex('my_col')
+```
+
+#### Defined in
+
+[index.ts:356](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L356)
 
 ___
 
@@ -157,7 +194,7 @@ await tbl.countRows() // Returns 1
 
 #### Defined in
 
-[index.ts:263](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L263)
+[index.ts:395](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L395)
 
 ___
 
@@ -183,7 +220,7 @@ Get statistics about an index.
 
 #### Defined in
 
-[index.ts:306](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L306)
+[index.ts:438](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L438)
 
 ___
 
@@ -203,7 +240,7 @@ List the indicies on this table.
 
 #### Defined in
 
-[index.ts:301](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L301)
+[index.ts:433](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L433)
 
 ___
 
@@ -213,7 +250,7 @@ ___
 
 #### Defined in
 
-[index.ts:195](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L195)
+[index.ts:277](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L277)
 
 ___
 
@@ -241,7 +278,17 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:217](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L217)
+[index.ts:299](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L299)
+
+___
+
+### schema
+
+• **schema**: `Promise`\<`Schema`\<`any`\>\>
+
+#### Defined in
+
+[index.ts:440](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L440)
 
 ___
 
@@ -267,7 +314,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:201](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L201)
+[index.ts:283](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L283)
 
 ___
 
@@ -305,8 +352,8 @@ const data = [
 const tbl = await con.createTable("my_table", data)
 
 await tbl.update({
-  filter: "id = 2",
-  updates: { vector: [2, 2], name: "Michael" },
+  where: "id = 2",
+  values: { vector: [2, 2], name: "Michael" },
 })
 
 let results = await tbl.search([1, 1]).execute();
@@ -318,4 +365,4 @@ let results = await tbl.search([1, 1]).execute();
 
 #### Defined in
 
-[index.ts:296](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L296)
+[index.ts:428](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L428)

docs/src/javascript/interfaces/UpdateArgs.md

@@ -20,7 +20,7 @@ new values to set
 
 #### Defined in
 
-[index.ts:320](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L320)
+[index.ts:454](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L454)
 
 ___
 
@@ -33,4 +33,4 @@ in which case all rows will be updated.
 
 #### Defined in
 
-[index.ts:314](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L314)
+[index.ts:448](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L448)

docs/src/javascript/interfaces/UpdateSqlArgs.md

@@ -20,7 +20,7 @@ new values to set as SQL expressions.
 
 #### Defined in
 
-[index.ts:334](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L334)
+[index.ts:468](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L468)
 
 ___
 
@@ -33,4 +33,4 @@ in which case all rows will be updated.
 
 #### Defined in
 
-[index.ts:328](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L328)
+[index.ts:462](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L462)

docs/src/javascript/interfaces/VectorIndex.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:338](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L338)
+[index.ts:472](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L472)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:339](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L339)
+[index.ts:473](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L473)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:340](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L340)
+[index.ts:474](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L474)

docs/src/javascript/interfaces/WriteOptions.md

@@ -24,4 +24,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation
 
 #### Defined in
 
-[index.ts:774](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L774)
+[index.ts:1015](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1015)

docs/src/javascript/modules.md

@@ -52,7 +52,7 @@
 
 #### Defined in
 
-[index.ts:755](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L755)
+[index.ts:996](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L996)
 
 ## Functions
 
@@ -60,29 +60,41 @@
 
 ▸ **connect**(`uri`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
-Connect to a LanceDB instance at the given URI
+Connect to a LanceDB instance at the given URI.
+
+Accpeted formats:
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (SaaS)
 
 #### Parameters
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. |
+| `uri` | `string` | The uri of the database. If the database uri starts with `db://` then it connects to a remote database. |
 
 #### Returns
 
 `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
+**`See`**
+
+[ConnectionOptions](interfaces/ConnectionOptions.md) for more details on the URI format.
+
 #### Defined in
 
-[index.ts:95](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L95)
+[index.ts:141](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L141)
 
 ▸ **connect**(`opts`): `Promise`\<[`Connection`](interfaces/Connection.md)\>
 
+Connect to a LanceDB instance with connection options.
+
 #### Parameters
 
-| Name | Type |
-| :------ | :------ |
-| `opts` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `opts` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> | The [ConnectionOptions](interfaces/ConnectionOptions.md) to use when connecting to the database. |
 
 #### Returns
 
@@ -90,7 +102,7 @@ Connect to a LanceDB instance at the given URI
 
 #### Defined in
 
-[index.ts:96](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L96)
+[index.ts:147](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L147)
 
 ___
 
@@ -110,4 +122,4 @@ value is WriteOptions
 
 #### Defined in
 
-[index.ts:781](https://github.com/lancedb/lancedb/blob/7856a94/node/src/index.ts#L781)
+[index.ts:1022](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1022)

docs/src/notebooks/code_qa_bot.ipynb

@@ -44,15 +44,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openai\n",
+    "from openai import OpenAI\n",
     "import os\n",
     "\n",
     "# Configuring the environment variable OPENAI_API_KEY\n",
     "if \"OPENAI_API_KEY\" not in os.environ:\n",
-    "    # OR set the key here as a variable\n",
-    "    openai.api_key = \"sk-...\"\n",
-    "    \n",
-    "assert len(openai.Model.list()[\"data\"]) > 0"
+    "    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+    "client = OpenAI()\n",
+    "assert len(client.models.list().data) > 0"
    ]
   },
   {

docs/src/notebooks/reproducibility.ipynb

@@ -10,18 +10,18 @@
     "Reproducibility is critical for AI. For code, it's easy to keep track of changes using Github or Gitlab.\n",
     "For data, it's not as easy. Most of the time, we're manually writing complicated data tracking code, wrestling with an external tool, and dealing with expensive duplicate snapshot copies with low granularity.\n",
     "\n",
-    "For vector databases, if we make a mistake, we have to blow away the index, correct the mistake, and then completely rebuild it. It's difficult to rollback mistakes and it destroys any historical paper trail to debug and diagnose errors.\n",
+    "While working with most other vector databases, if we loaded in the wrong data (or any other such mistakes), we have to blow away the index, correct the mistake, and then completely rebuild it. It's **really difficult** to rollback to an earlier state, and any such corrective action **destroys historical data and evidence**, which may be useful down the line to debug and diagnose issues.\n",
     "\n",
-    "LanceDB is the first and only vector database that supports full reproducibility natively.\n",
-    "Taking advantage of Lance columnar format, LanceDB supports:\n",
-    "- automatic versioning\n",
-    "- instance rollback\n",
-    "- appends, updates, deletions\n",
-    "- schema evolution\n",
+    "To our knowledge, LanceDB is the first and only vector database that supports full reproducibility and rollbacks natively.\n",
+    "Taking advantage of the Lance columnar data format, LanceDB supports:\n",
+    "- Automatic versioning\n",
+    "- Instant rollback\n",
+    "- Appends, updates, deletions\n",
+    "- Schema evolution\n",
     "\n",
-    "To make auditing, tracking, and reproducibility a breeze.\n",
+    "This makes auditing, tracking, and reproducibility a breeze!\n",
     "\n",
-    "Let's see how this all works"
+    "Let's see how this all works."
    ]
   },
   {

docs/src/notebooks/tables_guide.ipynb

@@ -13,7 +13,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 2,
    "id": "c1b4e34b-a49c-471d-a343-a5940bb5138a",
    "metadata": {},
    "outputs": [],
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "id": "4e5a8d07-d9a1-48c1-913a-8e0629289579",
    "metadata": {},
    "outputs": [],
@@ -44,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "id": "5df12f66-8d99-43ad-8d0b-22189ec0a6b9",
    "metadata": {},
    "outputs": [
@@ -62,7 +62,7 @@
        "long: [[-122.7,-74.1]]"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -90,7 +90,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
    "id": "f4d87ae9-0ccb-48eb-b31d-bb8f2370e47e",
    "metadata": {},
    "outputs": [
@@ -108,7 +108,7 @@
        "long: [[-122.7,-74.1]]"
       ]
      },
-     "execution_count": 3,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -135,10 +135,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "id": "25f34bcf-fca0-4431-8601-eac95d1bd347",
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[2024-01-31T18:59:33Z WARN  lance::dataset] No existing dataset at /Users/qian/Work/LanceDB/lancedb/docs/src/notebooks/.lancedb/table3.lance, it will be created\n"
+     ]
+    },
     {
      "data": {
       "text/plain": [
@@ -148,7 +155,7 @@
        "long: float"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -171,45 +178,51 @@
    "id": "4df51925-7ca2-4005-9c72-38b3d26240c6",
    "metadata": {},
    "source": [
-    "### From PyArrow Tables\n",
+    "### From an Arrow Table\n",
     "\n",
-    "You can also create LanceDB tables directly from pyarrow tables"
+    "You can also create LanceDB tables directly from pyarrow tables. LanceDB supports float16 type."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 7,
    "id": "90a880f6-be43-4c9d-ba65-0b05197c0f6f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "vector: fixed_size_list<item: float>[2]\n",
-       "  child 0, item: float\n",
-       "item: string\n",
-       "price: double"
+       "vector: fixed_size_list<item: halffloat>[16]\n",
+       "  child 0, item: halffloat\n",
+       "text: string"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "table = pa.Table.from_arrays(\n",
-    "        [\n",
-    "            pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
-    "                    pa.list_(pa.float32(), 2)),\n",
-    "            pa.array([\"foo\", \"bar\"]),\n",
-    "            pa.array([10.0, 20.0]),\n",
-    "        ],\n",
-    "        [\"vector\", \"item\", \"price\"],\n",
-    "    )\n",
+    "import numpy as np\n",
     "\n",
-    "db = lancedb.connect(\"db\")\n",
+    "dim = 16\n",
+    "total = 2\n",
+    "schema = pa.schema(\n",
+    "    [\n",
+    "        pa.field(\"vector\", pa.list_(pa.float16(), dim)),\n",
+    "        pa.field(\"text\", pa.string())\n",
+    "    ]\n",
+    ")\n",
+    "data = pa.Table.from_arrays(\n",
+    "    [\n",
+    "        pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],\n",
+    "                pa.list_(pa.float16(), dim)),\n",
+    "        pa.array([\"foo\", \"bar\"])\n",
+    "    ],\n",
+    "    [\"vector\", \"text\"],\n",
+    ")\n",
     "\n",
-    "tbl = db.create_table(\"test1\", table, mode=\"overwrite\")\n",
+    "tbl = db.create_table(\"f16_tbl\", data, schema=schema)\n",
     "tbl.schema"
    ]
   },
@@ -225,7 +238,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "id": "d81121d7-e4b7-447c-a48c-974b6ebb464a",
    "metadata": {},
    "outputs": [
@@ -240,7 +253,7 @@
        "imdb_id: int64 not null"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -282,7 +295,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "id": "bc247142-4e3c-41a2-b94c-8e00d2c2a508",
    "metadata": {},
    "outputs": [
@@ -292,7 +305,7 @@
        "LanceTable(table4)"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -333,7 +346,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 10,
    "id": "25ad3523-e0c9-4c28-b3df-38189c4e0e5f",
    "metadata": {},
    "outputs": [
@@ -346,7 +359,7 @@
        "price: double not null"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -385,7 +398,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 11,
    "id": "2814173a-eacc-4dd8-a64d-6312b44582cc",
    "metadata": {},
    "outputs": [],
@@ -411,7 +424,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 12,
    "id": "df9e13c0-41f6-437f-9dfa-2fd71d3d9c45",
    "metadata": {},
    "outputs": [
@@ -421,7 +434,7 @@
        "['table6', 'table4', 'table5', 'movielens_small']"
       ]
      },
-     "execution_count": 18,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -432,7 +445,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 13,
    "id": "9343f5ad-6024-42ee-ac2f-6c1471df8679",
    "metadata": {},
    "outputs": [
@@ -541,7 +554,7 @@
        "9  [5.9, 26.5]  bar   20.0"
       ]
      },
-     "execution_count": 20,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -564,7 +577,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 14,
    "id": "8a56250f-73a1-4c26-a6ad-5c7a0ce3a9ab",
    "metadata": {},
    "outputs": [],
@@ -590,7 +603,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 15,
    "id": "030c7057-b98e-4e2f-be14-b8c1f927f83c",
    "metadata": {},
    "outputs": [],
@@ -621,7 +634,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 16,
    "id": "e7a17de2-08d2-41b7-bd05-f63d1045ab1f",
    "metadata": {},
    "outputs": [
@@ -629,16 +642,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "32\n"
+      "22\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "17"
+       "12"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -661,7 +674,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 17,
    "id": "fe3310bd-08f4-4a22-a63b-b3127d22f9f7",
    "metadata": {},
    "outputs": [
@@ -681,25 +694,20 @@
       "8    [3.1, 4.1]   foo   10.0\n",
       "9    [3.1, 4.1]   foo   10.0\n",
       "10   [3.1, 4.1]   foo   10.0\n",
-      "11   [3.1, 4.1]   foo   10.0\n",
-      "12   [3.1, 4.1]   foo   10.0\n",
-      "13   [3.1, 4.1]   foo   10.0\n",
-      "14   [3.1, 4.1]   foo   10.0\n",
-      "15   [3.1, 4.1]   foo   10.0\n",
-      "16   [3.1, 4.1]   foo   10.0\n"
+      "11   [3.1, 4.1]   foo   10.0\n"
      ]
     },
     {
      "ename": "OSError",
-     "evalue": "LanceError(IO): Error during planning: column foo does not exist",
+     "evalue": "LanceError(IO): Error during planning: column foo does not exist, /Users/runner/work/lance/lance/rust/lance-core/src/error.rs:212:23",
      "output_type": "error",
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[30], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m tbl\u001b[38;5;241m.\u001b[39mto_pandas()\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/python/lancedb/table.py:610\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 610\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m~/Documents/lancedb/lancedb/env/lib/python3.11/site-packages/lance/dataset.py:489\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    487\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    488\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 489\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist"
+      "Cell \u001b[0;32mIn[17], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m to_remove \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mstr\u001b[39m(v) \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m to_remove)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(tbl\u001b[38;5;241m.\u001b[39mto_pandas())\n\u001b[0;32m----> 4\u001b[0m \u001b[43mtbl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mitem IN (\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mto_remove\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Work/LanceDB/lancedb/docs/doc-venv/lib/python3.11/site-packages/lancedb/table.py:872\u001b[0m, in \u001b[0;36mLanceTable.delete\u001b[0;34m(self, where)\u001b[0m\n\u001b[1;32m    871\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdelete\u001b[39m(\u001b[38;5;28mself\u001b[39m, where: \u001b[38;5;28mstr\u001b[39m):\n\u001b[0;32m--> 872\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwhere\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/Work/LanceDB/lancedb/docs/doc-venv/lib/python3.11/site-packages/lance/dataset.py:596\u001b[0m, in \u001b[0;36mLanceDataset.delete\u001b[0;34m(self, predicate)\u001b[0m\n\u001b[1;32m    594\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(predicate, pa\u001b[38;5;241m.\u001b[39mcompute\u001b[38;5;241m.\u001b[39mExpression):\n\u001b[1;32m    595\u001b[0m     predicate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(predicate)\n\u001b[0;32m--> 596\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdelete\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpredicate\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mOSError\u001b[0m: LanceError(IO): Error during planning: column foo does not exist, /Users/runner/work/lance/lance/rust/lance-core/src/error.rs:212:23"
      ]
     }
    ],
@@ -712,7 +720,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": null,
    "id": "87d5bc21-847f-4c81-b56e-f6dbe5d05aac",
    "metadata": {},
    "outputs": [],
@@ -729,7 +737,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": null,
    "id": "9cba4519-eb3a-4941-ab7e-873d762e750f",
    "metadata": {},
    "outputs": [],
@@ -742,7 +750,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": null,
    "id": "5bdc9801-d5ed-4871-92d0-88b27108e788",
    "metadata": {},
    "outputs": [
@@ -817,7 +825,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -27,11 +27,11 @@
      "output_type": "stream",
      "text": [
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
       "\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m23.1.1\u001B[0m\n",
-      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n"
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
      ]
     }
    ],
@@ -206,15 +206,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openai\n",
+    "from openai import OpenAI\n",
     "import os\n",
     "\n",
     "# Configuring the environment variable OPENAI_API_KEY\n",
     "if \"OPENAI_API_KEY\" not in os.environ:\n",
     "    # OR set the key here as a variable\n",
-    "    openai.api_key = \"sk-...\"\n",
+    "    os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
     "    \n",
-    "assert len(openai.Model.list()[\"data\"]) > 0"
+    "client = OpenAI()\n",
+    "assert len(client.models.list().data) > 0"
    ]
   },
   {
@@ -234,8 +235,8 @@
    "outputs": [],
    "source": [
     "def embed_func(c):    \n",
-    "    rs = openai.Embedding.create(input=c, engine=\"text-embedding-ada-002\")\n",
-    "    return [record[\"embedding\"] for record in rs[\"data\"]]"
+    "    rs = client.embeddings.create(input=c, model=\"text-embedding-ada-002\")\n",
+    "    return [rs.data[0].embedding]"
    ]
   },
   {
@@ -536,9 +537,8 @@
    ],
    "source": [
     "def complete(prompt):\n",
-    "    # query text-davinci-003\n",
-    "    res = openai.Completion.create(\n",
-    "        engine='text-davinci-003',\n",
+    "    res = client.completions.create(\n",
+    "        model='text-davinci-003',\n",
     "        prompt=prompt,\n",
     "        temperature=0,\n",
     "        max_tokens=400,\n",
@@ -547,7 +547,7 @@
     "        presence_penalty=0,\n",
     "        stop=None\n",
     "    )\n",
-    "    return res['choices'][0]['text'].strip()\n",
+    "    return res.choices[0].text\n",
     "\n",
     "# check that it works\n",
     "query = \"who was the 12th person on the moon and when did they land?\"\n",

docs/src/python/duckdb.md

@@ -1,14 +1,14 @@
 # DuckDB
 
-`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
+LanceDB is very well-integrated with [DuckDB](https://duckdb.org/), an in-process SQL OLAP database. This integration is done via [Arrow](https://duckdb.org/docs/guides/python/sql_on_arrow) .
 
-Let us start with installing `duckdb` and `lancedb`.
+We can demonstrate this by first installing `duckdb` and `lancedb`.
 
 ```shell
 pip install duckdb lancedb
 ```
 
-We will re-use [the dataset created previously](./arrow.md):
+We will re-use the dataset [created previously](./pandas_and_pyarrow.md):
 
 ```python
 import lancedb
@@ -22,7 +22,7 @@ table = db.create_table("pd_table", data=data)
 arrow_table = table.to_arrow()
 ```
 
-`DuckDB` can directly query the `arrow_table`:
+DuckDB can directly query the `pyarrow.Table` object:
 
 ```python
 import duckdb
@@ -40,6 +40,8 @@ duckdb.query("SELECT * FROM arrow_table")
 └─────────────┴─────────┴────────┘
 ```
 
+You can very easily run any other DuckDB SQL queries on your data.
+
 ```py
 duckdb.query("SELECT mean(price) FROM arrow_table")
 ```

docs/src/python/integration.md

@@ -1,7 +0,0 @@
-# Integration
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is very easy to be integrate with Python ecosystems.
-
-* [Pandas and Arrow Integration](./arrow.md)
-* [DuckDB Integration](./duckdb.md)

docs/src/python/pandas_and_pyarrow.md

@@ -1,13 +1,12 @@
 # Pandas and PyArrow
 
-
-Built on top of [Apache Arrow](https://arrow.apache.org/),
-`LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
-and PyArrow.
+Because Lance is built on top of [Apache Arrow](https://arrow.apache.org/),
+LanceDB is tightly integrated with the Python data ecosystem, including [Pandas](https://pandas.pydata.org/)
+and PyArrow. The sequence of steps in a typical workflow is shown below.
 
 ## Create dataset
 
-First, we need to connect to a `LanceDB` database.
+First, we need to connect to a LanceDB database.
 
 ```py
 
@@ -16,7 +15,7 @@ import lancedb
 db = lancedb.connect("data/sample-lancedb")
 ```
 
-Afterwards, we write a `Pandas DataFrame` to LanceDB directly.
+We can load a Pandas `DataFrame` to LanceDB directly.
 
 ```py
 import pandas as pd
@@ -29,11 +28,10 @@ data = pd.DataFrame({
 table = db.create_table("pd_table", data=data)
 ```
 
-Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
-[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
+Similar to the [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html) method, LanceDB's
+[`db.create_table()`](python.md/#lancedb.db.DBConnection.create_table) accepts data in a variety of forms.
 
-For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
-to lazily generate data:
+If you have a dataset that is larger than memory, you can create a table with `Iterator[pyarrow.RecordBatch]` to lazily load the data:
 
 ```py
 
@@ -59,13 +57,13 @@ schema=pa.schema([
 table = db.create_table("iterable_table", data=make_batches(), schema=schema)
 ```
 
-You will find detailed instructions of creating dataset in
-[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
+You will find detailed instructions of creating a LanceDB dataset in
+[Getting Started](../basic.md#quick-start) and [API](python.md/#lancedb.db.DBConnection.create_table)
 sections.
 
-## Vector Search
+## Vector search
 
-We can now perform similarity search via `LanceDB` Python API.
+We can now perform similarity search via the LanceDB Python API.
 
 ```py
 # Open the table previously created.
@@ -82,8 +80,8 @@ print(df)
 0  [5.9, 26.5]  bar   20.0  14257.05957
 ```
 
-If you have a simple filter, it's faster to provide a `where clause` to `LanceDB`'s search query.
-If you have more complex criteria, you can always apply the filter to the resulting Pandas `DataFrame`.
+If you have a simple filter, it's faster to provide a `where` clause to LanceDB's `search` method.
+For more complex filters or aggregations, you can always resort to using the underlying `DataFrame` methods after performing a search.
 
 ```python
 
@@ -97,4 +95,4 @@ df = results = table.search([100, 100]).to_pandas()
 results = df[df.price < 15]
 assert len(results) == 1
 assert results["item"].iloc[0] == "foo"
-```
+```

docs/src/python/polars_arrow.md

@@ -0,0 +1,142 @@
+# Polars
+
+LanceDB supports [Polars](https://github.com/pola-rs/polars), a blazingly fast DataFrame library for Python written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow under the hood. A deeper integration between Lance Tables and Polars DataFrames is in progress, but at the moment, you can read a Polars DataFrame into LanceDB and output the search results from a query to a Polars DataFrame.
+
+## Create & Query LanceDB Table
+
+### From Polars DataFrame
+
+First, we connect to a LanceDB database.
+
+```py
+import lancedb
+
+db = lancedb.connect("data/polars-lancedb")
+```
+
+We can load a Polars `DataFrame` to LanceDB directly.
+
+```py
+import polars as pl
+
+data = pl.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pl_table", data=data)
+```
+
+We can now perform similarity search via the LanceDB Python API.
+
+```py
+query = [3.0, 4.0]
+result = table.search(query).limit(1).to_polars()
+print(result)
+print(type(result))
+```
+
+In addition to the selected columns, LanceDB also returns a vector
+and also the `_distance` column which is the distance between the query
+vector and the returned vector.
+
+```
+shape: (1, 4)
+┌───────────────┬──────┬───────┬───────────┐
+│ vector        ┆ item ┆ price ┆ _distance │
+│ ---           ┆ ---  ┆ ---   ┆ ---       │
+│ array[f32, 2] ┆ str  ┆ f64   ┆ f32       │
+╞═══════════════╪══════╪═══════╪═══════════╡
+│ [3.1, 4.1]    ┆ foo  ┆ 10.0  ┆ 0.0       │
+└───────────────┴──────┴───────┴───────────┘
+<class 'polars.dataframe.frame.DataFrame'>
+```
+
+Note that the type of the result from a table search is a Polars DataFrame.
+
+### From Pydantic Models
+
+Alternately, we can create an empty LanceDB Table using a Pydantic schema and populate it with a Polars DataFrame.
+
+```py
+import polars as pl
+from lancedb.pydantic import Vector, LanceModel
+
+
+class Item(LanceModel):
+    vector: Vector(2)
+    item: str
+    price: float
+
+data = {
+    "vector": [[3.1, 4.1]],
+    "item": "foo",
+    "price": 10.0,
+}
+
+table = db.create_table("test_table", schema=Item)
+df = pl.DataFrame(data)
+# Add Polars DataFrame to table
+table.add(df)
+```
+
+The table can now be queried as usual.
+
+```py
+result = table.search([3.0, 4.0]).limit(1).to_polars()
+print(result)
+print(type(result))
+```
+
+```
+shape: (1, 4)
+┌───────────────┬──────┬───────┬───────────┐
+│ vector        ┆ item ┆ price ┆ _distance │
+│ ---           ┆ ---  ┆ ---   ┆ ---       │
+│ array[f32, 2] ┆ str  ┆ f64   ┆ f32       │
+╞═══════════════╪══════╪═══════╪═══════════╡
+│ [3.1, 4.1]    ┆ foo  ┆ 10.0  ┆ 0.02      │
+└───────────────┴──────┴───────┴───────────┘
+<class 'polars.dataframe.frame.DataFrame'>
+```
+
+This result is the same as the previous one, with a DataFrame returned.
+
+## Dump Table to LazyFrame
+
+As you iterate on your application, you'll likely need to work with the whole table's data pretty frequently.
+LanceDB tables can also be converted directly into a polars LazyFrame for further processing.
+
+```python
+ldf = table.to_polars()
+print(type(ldf))
+```
+
+Unlike the search result from a query, we can see that the type of the result is a LazyFrame.
+
+```
+<class 'polars.lazyframe.frame.LazyFrame'>
+```
+
+We can now work with the LazyFrame as we would in Polars, and collect the first result.
+
+```python
+print(ldf.first().collect())
+```
+
+```
+shape: (1, 3)
+┌───────────────┬──────┬───────┐
+│ vector        ┆ item ┆ price │
+│ ---           ┆ ---  ┆ ---   │
+│ array[f32, 2] ┆ str  ┆ f64   │
+╞═══════════════╪══════╪═══════╡
+│ [3.1, 4.1]    ┆ foo  ┆ 10.0  │
+└───────────────┴──────┴───────┘
+```
+
+The reason it's beneficial to not convert the LanceDB Table
+to a DataFrame is because the table can potentially be way larger
+than memory, and Polars LazyFrames allow us to work with such
+larger-than-memory datasets by not loading it into memory all at once.
+

docs/src/python/pydantic.md

@@ -7,7 +7,7 @@ LanceDB integrates with Pydantic for schema inference, data ingestion, and query
 
 LanceDB supports to create Apache Arrow Schema from a
 [Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
-via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
+via [pydantic_to_schema()](python.md#lancedb.pydantic.pydantic_to_schema) method.
 
 ::: lancedb.pydantic.pydantic_to_schema
 

docs/src/python/python.md

@@ -1,4 +1,6 @@
-# LanceDB Python API Reference
+# Python API Reference
+
+This section contains the API reference for the OSS Python API.
 
 ## Installation
 
@@ -36,7 +38,7 @@ pip install lancedb
 
 ::: lancedb.embeddings.open_clip.OpenClipEmbeddings
 
-::: lancedb.embeddings.with_embeddings
+::: lancedb.embeddings.utils.with_embeddings
 
 ## Context
 
@@ -56,6 +58,8 @@ pip install lancedb
 
 ::: lancedb.schema.vector
 
+::: lancedb.merge.LanceMergeInsertBuilder
+
 ## Integrations
 
 ### Pydantic

docs/src/python/saas-python.md

@@ -1,4 +1,6 @@
-# LanceDB Python API Reference
+# Python API Reference (SaaS)
+
+This section contains the API reference for the SaaS Python API.
 
 ## Installation
 

docs/src/search.md

@@ -1,33 +1,28 @@
 # Vector Search
 
-`Vector Search` finds the nearest vectors from the database.
-In a recommendation system or search engine, you can find similar products from
-the one you searched.
-In LLM and other AI applications,
-each data point can be [presented by the embeddings generated from some models](embeddings/index.md),
-it returns the most relevant features.
+A vector search finds the approximate or exact nearest neighbors to a given query vector.
 
-A search in high-dimensional vector space, is to find `K-Nearest-Neighbors (KNN)` of the query vector.
+- In a recommendation system or search engine, you can find similar records to
+  the one you searched.
+- In LLM and other AI applications,
+  each data point can be represented by [embeddings generated from existing models](embeddings/index.md),
+  following which the search returns the most relevant features.
 
-## Metric
+## Distance metrics
 
-In LanceDB, a `Metric` is the way to describe the distance between a pair of vectors.
-Currently, we support the following metrics:
+Distance metrics are a measure of the similarity between a pair of vectors.
+Currently, LanceDB supports the following metrics:
 
-| Metric      | Description                          |
-| ----------- | ------------------------------------ |
-| `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
-| `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
-| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
+| Metric   | Description                                                                 |
+| -------- | --------------------------------------------------------------------------- |
+| `l2`     | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
+| `cosine` | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)        |
+| `dot`    | [Dot Production](https://en.wikipedia.org/wiki/Dot_product)                 |
 
+## Exhaustive search (kNN)
 
-## Search
-
-### Flat Search
-
-If you do not create a vector index, LanceDB would need to exhaustively scan the entire vector column (via `Flat Search`)
-and compute the distance for *every* vector in order to find the closest matches. This is effectively a KNN search.
-
+If you do not create a vector index, LanceDB exhaustively scans the _entire_ vector space
+and compute the distance to every vector in order to find the exact nearest neighbors. This is effectively a kNN search.
 
 <!-- Setup Code
 ```python
@@ -42,21 +37,9 @@ data = [{"vector": row, "item": f"item {i}"}
 db.create_table("my_vectors", data=data)
 ```
 -->
-<!-- Setup Code
-```javascript
-const vectordb_setup = require('vectordb')
-const db_setup = await vectordb_setup.connect('data/sample-lancedb')
 
-let data = []
-for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-}
-await db_setup.createTable('my_vectors', data)
-```
--->
 === "Python"
 
-
     ```python
     import lancedb
     import numpy as np
@@ -73,18 +56,13 @@ await db_setup.createTable('my_vectors', data)
 === "JavaScript"
 
     ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
+    --8<-- "docs/src/search_legacy.ts:import"
 
-    const tbl = await db.openTable("my_vectors")
-
-    const results_1 = await tbl.search(Array(1536).fill(1.2))
-        .limit(10)
-        .execute()
+    --8<-- "docs/src/search_legacy.ts:search1"
     ```
 
-By default, `l2` will be used as `Metric` type. You can customize the metric type
-as well.
+By default, `l2` will be used as metric type. You can specify the metric type as
+`cosine` or `dot` if required.
 
 === "Python"
 
@@ -95,124 +73,115 @@ as well.
         .to_list()
     ```
 
-
 === "JavaScript"
 
     ```javascript
-    const results_2 = await tbl.search(Array(1536).fill(1.2))
-        .metricType("cosine")
-        .limit(10)
-        .execute()
+    --8<-- "docs/src/search_legacy.ts:search2"
     ```
 
+## Approximate nearest neighbor (ANN) search
 
-### Approximate Nearest Neighbor (ANN) Search with Vector Index.
+To perform scalable vector retrieval with acceptable latencies, it's common to build a vector index.
+While the exhaustive search is guaranteed to always return 100% recall, the approximate nature of
+an ANN search means that using an index often involves a trade-off between recall and latency.
 
-To accelerate vector retrievals, it is common to build vector indices.
-A vector index is a data structure specifically designed to efficiently organize and
-search vector data based on their similarity via the chosen distance metric.
-By constructing a vector index, you can reduce the search space and avoid the need
-for brute-force scanning of the entire vector column.
+See the [IVF_PQ index](./concepts/index_ivfpq.md.md) for a deeper description of how `IVF_PQ`
+indexes work in LanceDB.
 
-However, fast vector search using indices often entails making a trade-off with accuracy to some extent.
-This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
-always returns 100% recall.
+## Output search results
 
-See [ANN Index](ann_indexes.md) for more details.
-
-
-### Output formats
-
-LanceDB returns results in many different formats commonly used in python.
+LanceDB returns vector search results via different formats commonly used in python.
 Let's create a LanceDB table with a nested schema:
 
-```python
-from datetime import datetime
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-import numpy as np
-from pydantic import BaseModel
-uri = "data/sample-lancedb-nested"
+=== "Python"
 
-class Metadata(BaseModel):
-    source: str
-    timestamp: datetime
+    ```python
 
-class Document(BaseModel):
-    content: str
-    meta: Metadata
+    from datetime import datetime
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    import numpy as np
+    from pydantic import BaseModel
+    uri = "data/sample-lancedb-nested"
 
-class LanceSchema(LanceModel):
-    id: str
-    vector: Vector(1536)
-    payload: Document
+    class Metadata(BaseModel):
+        source: str
+        timestamp: datetime
 
-# Let's add 100 sample rows to our dataset
-data = [LanceSchema(
-    id=f"id{i}",
-    vector=np.random.randn(1536),
-    payload=Document(
-        content=f"document{i}", meta=Metadata(source=f"source{i%10}", timestamp=datetime.now())
-    ),
-) for i in range(100)]
+    class Document(BaseModel):
+        content: str
+        meta: Metadata
 
-tbl = db.create_table("documents", data=data)
-```
+    class LanceSchema(LanceModel):
+        id: str
+        vector: Vector(1536)
+        payload: Document
 
-#### As a pyarrow table
+    # Let's add 100 sample rows to our dataset
+    data = [LanceSchema(
+        id=f"id{i}",
+        vector=np.random.randn(1536),
+        payload=Document(
+            content=f"document{i}", meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now())
+        ),
+    ) for i in range(100)]
 
-Using `to_arrow()` we can get the results back as a pyarrow Table.
-This result table has the same columns as the LanceDB table, with 
-the addition of an `_distance` column for vector search or a `score`
-column for full text search.
+    tbl = db.create_table("documents", data=data)
+    ```
 
-```python
-tbl.search(np.random.randn(1536)).to_arrow()
-```
+    ### As a PyArrow table
 
-#### As a pandas dataframe
+    Using `to_arrow()` we can get the results back as a pyarrow Table.
+    This result table has the same columns as the LanceDB table, with
+    the addition of an `_distance` column for vector search or a `score`
+    column for full text search.
 
-You can also get the results as a pandas dataframe.
+    ```python
+    tbl.search(np.random.randn(1536)).to_arrow()
+    ```
 
-```python
-tbl.search(np.random.randn(1536)).to_pandas()
-```
+    ### As a Pandas DataFrame
 
-While other formats like Arrow/Pydantic/Python dicts have a natural 
-way to handle nested schemas, pandas can only store nested data as a 
-python dict column, which makes it difficult to support nested references.
-So for convenience, you can also tell LanceDB to flatten a nested schema 
-when creating the pandas dataframe. 
+    You can also get the results as a pandas dataframe.
 
-```python
-tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
-```
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas()
+    ```
 
-If your table has a deeply nested struct, you can control how many levels
-of nesting to flatten by passing in a positive integer.
+    While other formats like Arrow/Pydantic/Python dicts have a natural
+    way to handle nested schemas, pandas can only store nested data as a
+    python dict column, which makes it difficult to support nested references.
+    So for convenience, you can also tell LanceDB to flatten a nested schema
+    when creating the pandas dataframe.
 
-```python
-tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
-```
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+    ```
 
+    If your table has a deeply nested struct, you can control how many levels
+    of nesting to flatten by passing in a positive integer.
 
-#### As a list of python dicts
+    ```python
+    tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+    ```
 
-You can of course return results as a list of python dicts.
+    ### As a list of Python dicts
 
-```python
-tbl.search(np.random.randn(1536)).to_list()
-```
+    You can of course return results as a list of python dicts.
 
-#### As a list of pydantic models
+    ```python
+    tbl.search(np.random.randn(1536)).to_list()
+    ```
 
-We can add data using pydantic models, and we can certainly
-retrieve results as pydantic models
+    ### As a list of Pydantic models
 
-```python
-tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
-```
+    We can add data using Pydantic models, and we can certainly
+    retrieve results as Pydantic models
 
-Note that in this case the extra `_distance` field is discarded since
-it's not part of the LanceSchema.
+    ```python
+    tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+    ```
+
+    Note that in this case the extra `_distance` field is discarded since
+    it's not part of the LanceSchema.
 

docs/src/search_legacy.ts

@@ -0,0 +1,41 @@
+// --8<-- [start:import]
+import * as lancedb from "vectordb";
+// --8<-- [end:import]
+import * as fs from "fs";
+
+async function setup() {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    });
+  }
+  await db.createTable("my_vectors", data);
+}
+
+async () => {
+  await setup();
+
+  // --8<-- [start:search1]
+  const db = await lancedb.connect("data/sample-lancedb");
+  const tbl = await db.openTable("my_vectors");
+
+  const results_1 = await tbl.search(Array(1536).fill(1.2)).limit(10).execute();
+  // --8<-- [end:search1]
+
+  // --8<-- [start:search2]
+  const results_2 = await tbl
+    .search(Array(1536).fill(1.2))
+    .metricType(lancedb.MetricType.Cosine)
+    .limit(10)
+    .execute();
+  // --8<-- [end:search2]
+
+  console.log("search: done");
+};

docs/src/sql.md

@@ -1,22 +1,14 @@
-# SQL filters
+# Filtering
 
-LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
-filters. It can be used during hybrid vector search, update, and deletion operations.
+## Pre and post-filtering
 
-Currently, Lance supports a growing list of expressions.
+LanceDB supports filtering of query results based on metadata fields. By default, post-filtering is
+performed on the top-k results returned by the vector search. However, pre-filtering is also an
+option that performs the filter prior to vector search. This can be useful to narrow down on
+the search space on a very large dataset to reduce query latency.
 
-* ``>``, ``>=``, ``<``, ``<=``, ``=``
-* ``AND``, ``OR``, ``NOT``
-* ``IS NULL``, ``IS NOT NULL``
-* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
-* ``IN``
-* ``LIKE``, ``NOT LIKE``
-* ``CAST``
-* ``regexp_match(column, pattern)``
-
-For example, the following filter string is acceptable:
 <!-- Setup Code
-```python 
+```python
 import lancedb
 import numpy as np
 uri = "data/sample-lancedb"
@@ -29,7 +21,7 @@ tbl = db.create_table("my_vectors", data=data)
 ```
 -->
 <!-- Setup Code
-```javascript 
+```javascript
 const vectordb = require('vectordb')
 const db = await vectordb.connect('data/sample-lancedb')
 
@@ -40,6 +32,43 @@ for (let i = 0; i < 10_000; i++) {
 const tbl = await db.createTable('myVectors', data)
 ```
 -->
+
+=== "Python"
+
+    ```py
+    result = (
+        tbl.search([0.5, 0.2])
+        .where("id = 10", prefilter=True)
+        .limit(1)
+        .to_arrow()
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    --8<-- "docs/src/sql_legacy.ts:search"
+    ```
+
+## SQL filters
+
+Because it's built on top of [DataFusion](https://github.com/apache/arrow-datafusion), LanceDB
+embraces the utilization of standard SQL expressions as predicates for filtering operations.
+It can be used during vector search, update, and deletion operations.
+
+Currently, Lance supports a growing list of SQL expressions.
+
+- `>`, `>=`, `<`, `<=`, `=`
+- `AND`, `OR`, `NOT`
+- `IS NULL`, `IS NOT NULL`
+- `IS TRUE`, `IS NOT TRUE`, `IS FALSE`, `IS NOT FALSE`
+- `IN`
+- `LIKE`, `NOT LIKE`
+- `CAST`
+- `regexp_match(column, pattern)`
+
+For example, the following filter string is acceptable:
+
 === "Python"
 
     ```python
@@ -51,29 +80,27 @@ const tbl = await db.createTable('myVectors', data)
 === "Javascript"
 
     ```javascript
-    await tbl.search(Array(1536).fill(0))
-       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
-       .execute()
+    --8<-- "docs/src/sql_legacy.ts:vec_search"
     ```
 
-
 If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
 you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
 path must be wrapped in backticks.
 
 === "SQL"
+
     ```sql
     `CUBE` = 10 AND `column name with space` IS NOT NULL
       AND `nested with space`.`inner with space` < 2
     ```
 
-!!! warning
-    Field names containing periods (``.``) are not supported.
+!!!warning "Field names containing periods (`.`) are not supported."
 
 Literals for dates, timestamps, and decimals can be written by writing the string
 value after the type name. For example
 
 === "SQL"
+
     ```sql
     date_col = date '2021-01-01'
     and timestamp_col = timestamp '2021-01-01 00:00:00'
@@ -83,49 +110,47 @@ value after the type name. For example
 For timestamp columns, the precision can be specified as a number in the type
 parameter. Microsecond precision (6) is the default.
 
-| SQL              | Time unit    |
-|------------------|--------------|
-| ``timestamp(0)`` | Seconds      |
-| ``timestamp(3)`` | Milliseconds |
-| ``timestamp(6)`` | Microseconds |
-| ``timestamp(9)`` | Nanoseconds  |
+| SQL            | Time unit    |
+| -------------- | ------------ |
+| `timestamp(0)` | Seconds      |
+| `timestamp(3)` | Milliseconds |
+| `timestamp(6)` | Microseconds |
+| `timestamp(9)` | Nanoseconds  |
 
 LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
 The mapping from SQL types to Arrow types is:
 
-| SQL type | Arrow type |
-|----------|------------|
-| ``boolean`` | ``Boolean`` |
-| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
-| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
-| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
-| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
-| ``float`` | ``Float32`` |
-| ``double`` | ``Float64`` |
-| ``decimal(precision, scale)`` | ``Decimal128`` |
-| ``date`` | ``Date32`` |
-| ``timestamp`` | ``Timestamp`` [^1] |
-| ``string`` | ``Utf8`` |
-| ``binary`` | ``Binary`` |
+| SQL type                                                  | Arrow type         |
+| --------------------------------------------------------- | ------------------ |
+| `boolean`                                                 | `Boolean`          |
+| `tinyint` / `tinyint unsigned`                            | `Int8` / `UInt8`   |
+| `smallint` / `smallint unsigned`                          | `Int16` / `UInt16` |
+| `int` or `integer` / `int unsigned` or `integer unsigned` | `Int32` / `UInt32` |
+| `bigint` / `bigint unsigned`                              | `Int64` / `UInt64` |
+| `float`                                                   | `Float32`          |
+| `double`                                                  | `Float64`          |
+| `decimal(precision, scale)`                               | `Decimal128`       |
+| `date`                                                    | `Date32`           |
+| `timestamp`                                               | `Timestamp` [^1]   |
+| `string`                                                  | `Utf8`             |
+| `binary`                                                  | `Binary`           |
 
 [^1]: See precision mapping in previous table.
 
-
 ## Filtering without Vector Search
 
 You can also filter your data without search.
 
 === "Python"
-      ```python
-      tbl.search().where("id=10").limit(10).to_arrow()
-      ```
+
+    ```python
+    tbl.search().where("id = 10").limit(10).to_arrow()
+    ```
 
 === "JavaScript"
-      ```javascript
-      await tbl.where('id=10').limit(10).execute()
-      ```
 
-!!! warning
-    If your table is large, this could potentially return a very large
-    amount of data. Please be sure to use a `limit` clause unless
-    you're sure you want to return the whole result set.
+    ```javascript
+    --8<---- "docs/src/sql_legacy.ts:sql_search"
+    ```
+
+!!!warning "If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set."