Bump version: 0.9.0-beta.5 → 0.9.0-beta.6

# Conflicts:
#	python/pyproject.toml

.bumpversion.cfg

@@ -1,12 +0,0 @@
-[bumpversion]
-current_version = 0.3.0
-commit = True
-message = Bump version: {current_version} → {new_version}
-tag = True
-tag_name = v{new_version}
-
-[bumpversion:file:node/package.json]
-
-[bumpversion:file:rust/ffi/node/Cargo.toml]
-
-[bumpversion:file:rust/vectordb/Cargo.toml]

.bumpversion.toml

@@ -0,0 +1,57 @@
+[tool.bumpversion]
+current_version = "0.5.2"
+parse = """(?x)
+    (?P<major>0|[1-9]\\d*)\\.
+    (?P<minor>0|[1-9]\\d*)\\.
+    (?P<patch>0|[1-9]\\d*)
+    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
+"""
+serialize = [
+    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
+    "{major}.{minor}.{patch}",
+]
+search = "{current_version}"
+replace = "{new_version}"
+regex = false
+ignore_missing_version = false
+ignore_missing_files = false
+tag = true
+sign_tags = false
+tag_name = "v{new_version}"
+tag_message = "Bump version: {current_version} → {new_version}"
+allow_dirty = true
+commit = true
+message = "Bump version: {current_version} → {new_version}"
+commit_args = ""
+
+[tool.bumpversion.parts.pre_l]
+values = ["beta", "final"]
+optional_value = "final"
+
+[[tool.bumpversion.files]]
+filename = "node/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+[[tool.bumpversion.files]]
+filename = "nodejs/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+# nodejs binary packages
+[[tool.bumpversion.files]]
+glob = "nodejs/npm/*/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+# Cargo files
+# ------------
+[[tool.bumpversion.files]]
+filename = "rust/ffi/node/Cargo.toml"
+search = "\nversion = \"{current_version}\""
+replace = "\nversion = \"{new_version}\""
+
+[[tool.bumpversion.files]]
+filename = "rust/lancedb/Cargo.toml"
+search = "\nversion = \"{current_version}\""
+replace = "\nversion = \"{new_version}\""

.cargo/config.toml

@@ -0,0 +1,40 @@
+[profile.release]
+lto = "fat"
+codegen-units = 1
+
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+# Prioritize compile time over runtime performance
+codegen-units = 16
+lto = "thin"
+
+[target.'cfg(all())']
+rustflags = [
+    "-Wclippy::all",
+    "-Wclippy::style",
+    "-Wclippy::fallible_impl_from",
+    "-Wclippy::manual_let_else",
+    "-Wclippy::redundant_pub_crate",
+    "-Wclippy::string_add_assign",
+    "-Wclippy::string_add",
+    "-Wclippy::string_lit_as_bytes",
+    "-Wclippy::string_to_string",
+    "-Wclippy::use_self",
+    "-Dclippy::cargo",
+    "-Dclippy::dbg_macro",
+    # not too much we can do to avoid multiple crate versions
+    "-Aclippy::multiple-crate-versions",
+    "-Aclippy::wildcard_dependencies",
+]
+
+[target.x86_64-unknown-linux-gnu]
+rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
+
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
+
+# Not all Windows systems have the C runtime installed, so this avoids library
+# not found errors on systems that are missing it.
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-Ctarget-feature=+crt-static"]

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Node / Typescript
+description: File a bug report
+title: "bug(node): "
+labels: [bug, typescript]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `npm list | grep vectordb`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Python
+description: File a bug report
+title: "bug(python): "
+labels: [bug, python]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Discord Community Support
+    url: https://discord.com/invite/zMM32dvNtd
+    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,23 @@
+name: 'Documentation improvement'
+description: Report an issue with the documentation.
+labels: [documentation]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the issue with the documentation and how it can be fixed or improved.
+    validations:
+      required: true
+
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: >
+        Provide a link to the existing documentation, if applicable.
+      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,31 @@
+name: Feature suggestion
+description: Suggestion a new feature for LanceDB
+title: "Feature: "
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Share a new idea for a feature or improvement. Be sure to search existing
+        issues first to avoid duplicates.
+  - type: dropdown
+    id: sdk
+    attributes:
+      label: SDK
+      description: Which SDK are you using? This helps us prioritize.
+      options:
+        - Python
+        - Node
+        - Rust
+      default: 0
+    validations:
+      required: false
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: |
+        Describe the feature and why it would be useful. If applicable, consider
+        providing a code example of what it might be like to use the feature.
+    validations:
+      required: true

.github/labeler.yml

@@ -0,0 +1,33 @@
+version: 1
+appendOnly: true
+# Labels are applied based on conventional commits standard
+# https://www.conventionalcommits.org/en/v1.0.0/
+# These labels are later used in release notes. See .github/release.yml
+labels:
+# If the PR title has an ! before the : it will be considered a breaking change
+# For example, `feat!: add new feature` will be considered a breaking change
+- label: breaking-change
+  title: "^[^:]+!:.*"
+- label: breaking-change
+  body: "BREAKING CHANGE"
+- label: enhancement
+  title: "^feat(\\(.+\\))?!?:.*"
+- label: bug
+  title: "^fix(\\(.+\\))?!?:.*"
+- label: documentation
+  title: "^docs(\\(.+\\))?!?:.*"
+- label: performance
+  title: "^perf(\\(.+\\))?!?:.*"
+- label: ci
+  title: "^ci(\\(.+\\))?!?:.*"
+- label: chore
+  title: "^(chore|test|build|style)(\\(.+\\))?!?:.*"
+- label: Python
+  files:
+    - "^python\\/.*"
+- label: Rust
+  files:
+    - "^rust\\/.*"
+- label: typescript
+  files:
+    - "^node\\/.*"

.github/release_notes.json

@@ -0,0 +1,41 @@
+{
+    "ignore_labels": ["chore"],
+    "pr_template": "- ${{TITLE}} by @${{AUTHOR}} in ${{URL}}",
+    "categories": [
+        {
+            "title": "## 🏆 Highlights",
+            "labels": ["highlight"]
+        },
+        {
+            "title": "## 🛠 Breaking Changes",
+            "labels": ["breaking-change"]
+        },
+        {
+            "title": "## ⚠️ Deprecations ",
+            "labels": ["deprecation"]
+        },
+        {
+            "title": "## 🎉 New Features",
+            "labels": ["enhancement"]
+        },
+        {
+            "title": "## 🐛 Bug Fixes",
+            "labels": ["bug"]
+        },
+        {
+            "title": "## 📚 Documentation",
+            "labels": ["documentation"]
+        },
+        {
+            "title": "## 🚀 Performance Improvements",
+            "labels": ["performance"]
+        },
+        {
+            "title": "## Other Changes"
+        },
+        {
+            "title": "## 🔧 Build and CI",
+            "labels": ["ci"]
+        }
+    ]
+}

.github/workflows/build_linux_wheel/action.yml

@@ -0,0 +1,63 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build-linux-wheel
+description: "Build a manylinux wheel for lance"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11, 12"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+  arm-build:
+    description: "Build for arm64 instead of x86_64"
+    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
+    required: false
+    default: "false"
+  manylinux:
+    description: "The manylinux version to build for"
+    required: false
+    default: "2_17"
+runs:
+  using: "composite"
+  steps:
+    - name: CONFIRM ARM BUILD
+      shell: bash
+      run: |
+        echo "ARM BUILD: ${{ inputs.arm-build }}"
+    - name: Build x86_64 Manylinux wheel
+      if: ${{ inputs.arm-build == 'false' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        target: x86_64-unknown-linux-gnu
+        manylinux: ${{ inputs.manylinux }}
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          yum install -y openssl-devel \
+            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip
+    - name: Build Arm Manylinux Wheel
+      if: ${{ inputs.arm-build == 'true' }}
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        target: aarch64-unknown-linux-gnu
+        manylinux: ${{ inputs.manylinux }}
+        args: ${{ inputs.args }}
+        before-script-linux: |
+          set -e
+          apt install -y unzip
+          if [ $(uname -m) = "x86_64" ]; then
+            PROTOC_ARCH="x86_64"
+          else
+            PROTOC_ARCH="aarch_64"
+          fi
+          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
+            && unzip /tmp/protoc.zip -d /usr/local \
+            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -0,0 +1,26 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install macos dependency
+      shell: bash
+      run: |
+        brew install protobuf
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        working-directory: python
+        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -0,0 +1,34 @@
+# We create a composite action to be re-used both for testing and for releasing
+name: build_wheel
+description: "Build a lance wheel"
+inputs:
+  python-minor-version:
+    description: "8, 9, 10, 11"
+    required: true
+  args:
+    description: "--release"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Install Protoc v21.12
+      working-directory: C:\
+      run: |
+        New-Item -Path 'C:\protoc' -ItemType Directory
+        Set-Location C:\protoc
+        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+        7z x protoc.zip
+        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+      shell: powershell
+    - name: Build wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        command: build
+        args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
+        working-directory: python
+    - uses: actions/upload-artifact@v4
+      with:
+        name: windows-wheels
+        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -1,13 +1,20 @@
 name: Cargo Publish
 
 on:
-  release:
-    types: [ published ]
+  push:
+    tags-ignore:
+      # We don't publish pre-releases for Rust. Crates.io is just a source
+      # distribution, so we don't need to publish pre-releases.
+      - 'v*-beta*'
+      - '*-v*' # for example, python-vX.Y.Z
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
+  # Up-to-date compilers needed for fp16kernels.
+  CC: gcc-12
+  CXX: g++-12
 
 jobs:
   build:
@@ -16,7 +23,7 @@ jobs:
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -26,4 +33,4 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Publish the package
         run: |
-          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/dev.yml

@@ -0,0 +1,81 @@
+name: PR Checks
+
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  labeler:
+    permissions:
+      pull-requests: write
+    name: Label PR
+    runs-on: ubuntu-latest
+    steps:
+      - uses: srvaroa/labeler@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  commitlint:
+    permissions:
+      pull-requests: write
+    name: Verify PR title / description conforms to semantic-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+      # These rules are disabled because Github will always ensure there
+      # is a blank line between the title and the body and Github will
+      # word wrap the description field to ensure a reasonable max line
+      # length.
+      - run: npm install @commitlint/config-conventional
+      - run: >
+          echo 'module.exports = {
+            "rules": {
+              "body-max-line-length": [0, "always", Infinity],
+              "footer-max-line-length": [0, "always", Infinity],
+              "body-leading-blank": [0, "always"]
+            }
+          }' > .commitlintrc.js
+      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+        env:
+          COMMIT_MSG: >
+            ${{ github.event.pull_request.title }}
+
+            ${{ github.event.pull_request.body }}
+      - if: failure()
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const message = `**ACTION NEEDED**
+              
+              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
+
+              The PR title and description are used as the merge commit message.\
+              Please update your PR title and description to match the specification.
+
+              For details on the error please inspect the "PR Title Check" action.
+              `
+            // Get list of current comments
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number
+            });
+            // Check if this job already commented
+            for (const comment of comments) {
+              if (comment.body === message) {
+                return // Already commented
+              }
+            }
+            // Post the comment about Conventional Commits
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: message
+            })
+            core.setFailed(message)

.github/workflows/docs.yml

@@ -24,12 +24,16 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-22.04
+    runs-on: buildjet-8vcpu-ubuntu-2204
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: Install dependecies needed for ubuntu
+        run: |
+          sudo apt install -y protobuf-compiler libssl-dev
+          rustup update && rustup default        
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: "pip"
@@ -42,7 +46,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - uses: Swatinem/rust-cache@v2
@@ -62,8 +66,9 @@ jobs:
         run: |
           npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
+        working-directory: docs
         run: |
-          PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
+          PYTHONPATH=. mkdocs build
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact

.github/workflows/docs_test.yml

@@ -18,26 +18,30 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-minor-version: [ "11" ]
-        os: ["ubuntu-22.04"]
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+    - name: Print CPU capabilities
+      run: cat /proc/cpuinfo
+    - name: Install dependecies needed for ubuntu
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+        rustup update && rustup default
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.${{ matrix.python-minor-version }}
+        python-version: 3.11
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
     - name: Build Python
       working-directory: docs/test
       run:
@@ -52,42 +56,45 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: ${{ matrix.os }}
+    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    timeout-minutes: 60
     strategy:
-      matrix:
-        node-version: [ "18" ]
-        os: ["ubuntu-22.04"]
+      fail-fast: false
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
+    - name: Print CPU capabilities
+      run: cat /proc/cpuinfo
     - name: Set up Node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: ${{ matrix.node-version }}
+        node-version: 20
     - name: Install dependecies needed for ubuntu
-      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
-    - name: Install node dependencies
-      run: |
-        cd docs/test
-        npm install 
+        rustup update && rustup default
     - name: Rust cache
       uses: swatinem/rust-cache@v2
-    - name: Install LanceDB
+    - name: Install node dependencies
       run: |
-        cd docs/test/node_modules/vectordb 
+        sudo swapoff -a
+        sudo fallocate -l 8G /swapfile
+        sudo chmod 600 /swapfile
+        sudo mkswap /swapfile
+        sudo swapon /swapfile
+        sudo swapon --show
+        cd node
         npm ci
         npm run build-release
-        npm run tsc
-    - name: Create test files
-      run: |
-        cd docs/test
-        node md_testing.js
+        cd ../docs
+        npm install
     - name: Test
+      env:
+        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
+        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
       run: |
-        cd docs/test/node
-        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done
+        cd docs
+        npm t

.github/workflows/java.yml

@@ -0,0 +1,85 @@
+name: Build and Run Java JNI Tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - java/**
+      - rust/**
+      - .github/workflows/java.yml
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  CARGO_BUILD_JOBS: "1"
+jobs:
+  linux-build:
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04 + Java 11 & 17
+    defaults:
+      run:
+        working-directory: ./java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: java/core/lancedb-jni
+      - name: Run cargo fmt
+        run: cargo fmt --check
+        working-directory: ./java/core/lancedb-jni
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: "maven"
+      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
+      - name: Install Java 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+          cache: "maven"
+      - name: Java Style Check
+        run: mvn checkstyle:check
+      # Disable because of issues in lancedb rust core code
+      # - name: Rust Clippy
+      #   working-directory: java/core/lancedb-jni
+      #   run: cargo clippy --all-targets -- -D warnings
+      - name: Running tests with Java 11
+        run: mvn clean test
+      - name: Running tests with Java 17
+        run: |
+          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
+          -XX:+IgnoreUnrecognizedVMOptions \
+          --add-opens=java.base/java.lang=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
+          --add-opens=java.base/java.io=ALL-UNNAMED \
+          --add-opens=java.base/java.net=ALL-UNNAMED \
+          --add-opens=java.base/java.nio=ALL-UNNAMED \
+          --add-opens=java.base/java.util=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
+          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
+          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
+          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
+          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
+          -Djdk.reflect.useDirectMethodHandle=false \
+          -Dio.netty.tryReflectionSetAccessible=true"
+          JAVA_HOME=$JAVA_17 mvn clean test

.github/workflows/make-release-commit.yml

@@ -1,59 +1,99 @@
 name: Create release commit
 
+# This workflow increments versions, tags the version, and pushes it.
+# When a tag is pushed, another workflow is triggered that creates a GH release
+# and uploads the binaries. This workflow is only for creating the tag.
+
+# This script will enforce that a minor version is incremented if there are any
+# breaking changes since the last minor increment. However, it isn't able to
+# differentiate between breaking changes in Node versus Python. If you wish to
+# bypass this check, you can manually increment the version and push the tag.
 on:
   workflow_dispatch:
     inputs:
       dry_run:
         description: 'Dry run (create the local commit/tags but do not push it)'
         required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
+        default: false
+        type: boolean
+      type:
         description: 'What kind of release is this?'
         required: true
-        default: 'patch'
+        default: 'preview'
         type: choice
         options:
-          - patch
-          - minor
-          - major
+          - preview
+          - stable
+      python:
+        description: 'Make a Python release'
+        required: true
+        default: true
+        type: boolean
+      other:
+        description: 'Make a Node/Rust release'
+        required: true
+        default: true
+        type: boolean
+      bump-minor:
+        description: 'Bump minor version'
+        required: true
+        default: false
+        type: boolean
 
 jobs:
-  bump-version:
+  make-release:
+    # Creates tag and GH release. The GH release will trigger the build and release jobs.
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
-      - name: Check out main
-        uses: actions/checkout@v3
+      - name: Output Inputs
+        run: echo "${{ toJSON(github.event.inputs) }}"
+      - uses: actions/checkout@v4
         with:
-          ref: main
-          persist-credentials: false
           fetch-depth: 0
           lfs: true
+          # It's important we use our token here, as the default token will NOT
+          # trigger any workflows watching for new tags. See:
+          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
-      - name: Bump version, create tag and commit
+          python-version: "3.11"
+      - name: Bump Python version
+        if: ${{ inputs.python }}
+        working-directory: python
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          pip install bump2version
-          bumpversion --verbose ${{ inputs.part }}
-      - name: Push new version and tag
-        if: ${{ inputs.dry_run }} == "false"
+          # Need to get the commit before bumping the version, so we can
+          # determine if there are breaking changes in the next step as well.
+          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+          pip install bump-my-version PyGithub packaging
+          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
+      - name: Bump Node/Rust version
+        if: ${{ inputs.other }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pip install bump-my-version PyGithub packaging
+          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+      - name: Push new version tag
+        if: ${{ !inputs.dry_run }}
         uses: ad-m/github-push-action@master
         with:
+          # Need to use PAT here too to trigger next workflow. See comment above.
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-          branch: main
+          branch: ${{ github.ref }}
           tags: true
       - uses: ./.github/workflows/update_package_lock
-        if: ${{ inputs.dry_run }} == "false"
+        if: ${{ !inputs.dry_run && inputs.other }}
         with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/node.yml

@@ -11,47 +11,33 @@ on:
       - .github/workflows/node.yml
       - docker-compose.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
+  #
+  # Use native CPU to accelerate tests if possible, especially for f16
+  # target-cpu=haswell fixes failing ci build
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        lfs: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18
-        cache: 'npm'
-        cache-dependency-path: node/package-lock.json
-    - name: Lint
-      run: |
-        npm ci
-        npm run lint
   linux:
     name: Linux (Node ${{ matrix.node-version }})
     timeout-minutes: 30
     strategy:
       matrix:
-        node-version: [ "16", "18" ]
+        node-version: [ "18", "20" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
@@ -68,7 +54,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz
@@ -84,13 +69,13 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 20
         cache: 'npm'
         cache-dependency-path: node/package-lock.json
     - uses: Swatinem/rust-cache@v2
@@ -99,7 +84,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz
@@ -123,14 +107,15 @@ jobs:
       AWS_ENDPOINT: http://localhost:4566
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
+      ALLOW_HTTP: true
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
         lfs: true
     - uses: actions/setup-node@v3
       with:
-        node-version: 18
+        node-version: 20
         cache: 'npm'
         cache-dependency-path: node/package-lock.json
     - name: start local stack
@@ -153,7 +138,6 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
         npm run pack-build
         npm install --no-save ./dist/lancedb-vectordb-*.tgz

.github/workflows/nodejs.yml

@@ -0,0 +1,122 @@
+name: NodeJS (NAPI)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - nodejs/**
+      - .github/workflows/nodejs.yml
+      - docker-compose.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: nodejs/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Lint
+      run: |
+        cargo fmt --all -- --check
+        cargo clippy --all --all-features -- -D warnings
+        npm ci
+        npm run lint-ci
+  linux:
+    name: Linux (NodeJS ${{ matrix.node-version }})
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        node-version: [ "18", "20" ]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Setup localstack
+      working-directory: .
+      run: docker compose up --detach --wait
+    - name: Test
+      env:
+        S3_TEST: "1"
+      run: npm run test
+  macos:
+    timeout-minutes: 30
+    runs-on: "macos-14"
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - uses: Swatinem/rust-cache@v2
+    - name: Install dependencies
+      run: |
+        brew install protobuf
+        npm install -g @napi-rs/cli
+    - name: Build
+      run: |
+        npm ci
+        npm run build
+    - name: Test
+      run: |
+        npm run test

.github/workflows/npm-publish.yml

@@ -1,8 +1,9 @@
 name: NPM Publish
 
 on:
-  release:
-    types: [ published ]
+  push:
+    tags:
+      - "v*"
 
 jobs:
   node:
@@ -15,11 +16,11 @@ jobs:
         working-directory: node
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -31,41 +32,72 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: node-package
           path: |
             node/vectordb-*.tgz
 
   node-macos:
-    runs-on: macos-12
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install system dependencies
         run: brew install protobuf
       - name: Install npm dependencies
         run: |
           cd node
           npm ci
-      - name: Install rustup target
-        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-        run: rustup target add aarch64-apple-darwin
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-darwin
+          name: node-native-darwin-${{ matrix.config.arch }}
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
 
+  nodejs-macos:
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-14
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build MacOS native nodejs modules
+        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-darwin-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
+
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
@@ -78,20 +110,85 @@ jobs:
           - arch: x86_64
             runner: ubuntu-latest
           - arch: aarch64
-            runner: buildjet-4vcpu-ubuntu-2204-arm
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: warp-ubuntu-latest-arm64-4x
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      # To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
       - name: Build Linux Artifacts
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-linux
+          name: node-native-linux-${{ matrix.config.arch }}
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
+  nodejs-linux:
+    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-linux-${{ matrix.config.arch }}
+          path: |
+            nodejs/dist/*.node
+      # The generic files are the same in all distros so we just pick
+      # one to do the upload.
+      - name: Upload Generic Artifacts
+        if: ${{ matrix.config.arch == 'x86_64' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-dist
+          path: |
+            nodejs/dist/*
+            !nodejs/dist/*.node
+
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -102,7 +199,7 @@ jobs:
         target: [x86_64-pc-windows-msvc]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -119,40 +216,132 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: native-windows
+          name: node-native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
+  nodejs-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd nodejs
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nodejs-native-windows
+          path: |
+            nodejs/dist/*.node
+
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: node-*
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
+          # Tag beta as "preview" instead of default "latest". See lancedb 
+          # npm publish step for more info.
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            PUBLISH_ARGS="--tag preview"
+          fi
+
           mv */*.tgz .
           for filename in *.tgz; do
-            npm publish $filename
+            npm publish $PUBLISH_ARGS $filename
           done
 
+  release-nodejs:
+    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: nodejs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: nodejs-dist
+          path: nodejs/dist
+      - uses: actions/download-artifact@v4
+        name: Download arch-specific binaries
+        with:
+          pattern: nodejs-*
+          path: nodejs/nodejs-artifacts
+          merge-multiple: true
+      - name: Display structure of downloaded files
+        run: find .
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+      - name: Install napi-rs
+        run: npm install -g @napi-rs/cli
+      - name: Prepare artifacts
+        run: npx napi artifacts -d nodejs-artifacts
+      - name: Display structure of staged files
+        run: find npm
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        # By default, things are published to the latest tag. This is what is
+        # installed by default if the user does not specify a version. This is
+        # good for stable releases, but for pre-releases, we want to publish to
+        # the "preview" tag so they can install with `npm install lancedb@preview`.
+        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
+        run: |
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            npm publish --access public --tag preview
+          else
+            npm publish --access public
+          fi
+
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false
@@ -160,4 +349,87 @@ jobs:
           lfs: true
       - uses: ./.github/workflows/update_package_lock
         with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  update-package-lock-nodejs:
+    needs: [release-nodejs]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Release Notes
+        id: release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: ${{ contains('beta', github.ref) }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.release_notes.outputs.changelog }}

.github/workflows/pypi-publish.yml

@@ -1,31 +1,166 @@
 name: PyPI Publish
 
 on:
-  release:
-    types: [ published ]
+  push:
+    tags:
+      - 'python-v*'
 
 jobs:
-  publish:
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the python-make-release action
-    if: startsWith(github.ref, 'refs/tags/python-v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
+  linux:
+    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        config:
+          - platform: x86_64
+            manylinux: "2_17"
+            extra_args: ""
+            runner: ubuntu-22.04
+          - platform: x86_64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+            runner: ubuntu-22.04
+          - platform: aarch64
+            manylinux: "2_17"
+            extra_args: ""
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: ubuntu-2404-8x-arm64
+          - platform: aarch64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+            runner: ubuntu-2404-8x-arm64
+    runs-on: ${{ matrix.config.runner }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
-      - name: Build distribution
-        run: |
-          ls -la
-          pip install wheel setuptools --upgrade
-          python setup.py sdist bdist_wheel
-      - name: Publish
-        uses: pypa/gh-action-pypi-publish@v1.8.5
+          python-version: 3.8
+      - uses: ./.github/workflows/build_linux_wheel
         with:
-          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          packages-dir: python/dist
+          python-minor-version: 8
+          args: "--release --strip ${{ matrix.config.extra_args }}"
+          arm-build: ${{ matrix.config.platform == 'aarch64' }}
+          manylinux: ${{ matrix.config.manylinux }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
+  mac:
+    timeout-minutes: 60
+    runs-on: ${{ matrix.config.runner }}
+    strategy:
+      matrix:
+        config:
+          - target: x86_64-apple-darwin
+            runner: macos-13
+          - target: aarch64-apple-darwin
+            runner: macos-14
+    env:
+      MACOSX_DEPLOYMENT_TARGET: 10.15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - uses: ./.github/workflows/build_mac_wheel
+        with:
+          python-minor-version: 8
+          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
+  windows:
+    timeout-minutes: 60
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - uses: ./.github/workflows/build_windows_wheel
+        with:
+          python-minor-version: 8
+          args: "--release --strip"
+          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
+      - uses: ./.github/workflows/upload_wheel
+        with:
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=python-v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^python-v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py python-v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^python-v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py python-v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Python Release Notes
+        id: python_release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create Python GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: ${{ contains('beta', github.ref) }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.python_release_notes.outputs.changelog }}

.github/workflows/python-make-release-commit.yml

@@ -1,56 +0,0 @@
-name: Python - Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Bump version, create tag and commit
-      working-directory: python
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/python.yml

@@ -8,64 +8,164 @@ on:
     paths:
       - python/**
       - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  linux:
+  lint:
+    name: "Lint"
     timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: [ "8", "9", "10", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-      uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-        python-version: 3.${{ matrix.python-minor-version }}
-    - name: Install lancedb
+          python-version: "3.11"
+      - name: Install ruff
         run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
-  mac:
+          pip install ruff==0.2.2
+      - name: Format check
+        run: ruff format --check .
+      - name: Lint
+        run: ruff .
+  doctest:
+    name: "Doctest"
     timeout-minutes: 30
-    runs-on: "macos-12"
+    runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-      uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-    - name: Install lancedb
+          cache: "pip"
+      - name: Install protobuf
         run: |
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - name: Install
+        run: |
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
+          pip install tantivy
+          pip install mlx
+      - name: Doctest
+        run: pytest --doctest-modules python/lancedb
+  linux:
+    name: "Linux: python-3.${{ matrix.python-minor-version }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: ["9", "11"]
+    runs-on: "ubuntu-22.04"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install protobuf
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.${{ matrix.python-minor-version }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_linux_wheel
+      - uses: ./.github/workflows/run_tests
+        with:
+          integration: true
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
+  platform:
+    name: "Mac: ${{ matrix.config.name }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        config:
+          - name: x86
+            runner: macos-13
+          - name: Arm
+            runner: macos-14
+    runs-on: "${{ matrix.config.runner }}"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_mac_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
+  windows:
+    name: "Windows: ${{ matrix.config.name }}"
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        config:
+          - name: x86
+            runner: windows-latest
+    runs-on: "${{ matrix.config.runner }}"
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: python
+      - uses: ./.github/workflows/build_windows_wheel
+      - uses: ./.github/workflows/run_tests
+      # Make sure wheels are not included in the Rust cache
+      - name: Delete wheels
+        run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
@@ -74,25 +174,22 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler
       - name: Set up Python
-      uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
-        pip install -e .[tests]
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
+          pip install tantivy
       - name: Run tests
-      run: pytest -m "not slow" -x -v --durations=30 tests
-    - name: doctest
-      run: pytest --doctest-modules lancedb
+        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -0,0 +1,31 @@
+name: run-tests
+
+description: "Install lance wheel and run unit tests"
+inputs:
+  python-minor-version:
+    required: true
+    description: "8 9 10 11 12"
+  integration:
+    required: false
+    description: "Run integration tests"
+    default: "false"
+runs:
+  using: "composite"
+  steps:
+    - name: Install lancedb
+      shell: bash
+      run: |
+        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
+    - name: Setup localstack for integration tests
+      if: ${{ inputs.integration == 'true' }}
+      shell: bash
+      working-directory: .
+      run: docker compose up --detach --wait
+    - name: pytest (with integration)
+      shell: bash
+      if: ${{ inputs.integration == 'true' }}
+      run: pytest -m "not slow" -x -v --durations=30 python/python/tests
+    - name: pytest (no integration tests)
+      shell: bash
+      if: ${{ inputs.integration != 'true' }}
+      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -10,6 +10,10 @@ on:
       - rust/**
       - .github/workflows/rust.yml
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
@@ -20,15 +24,19 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
-  linux:
+  lint:
     timeout-minutes: 30
     runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
         working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
           fetch-depth: 0
           lfs: true
@@ -39,19 +47,54 @@ jobs:
       run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
+    - name: Run format
+      run: cargo fmt --all -- --check
+    - name: Run clippy
+      run: cargo clippy --all --all-features -- -D warnings
+  linux:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Start S3 integration test environment
+      working-directory: .
+      run: docker compose up --detach --wait
     - name: Build
       run: cargo build --all-features
     - name: Run tests
       run: cargo test --all-features
+    - name: Run examples
+      run: cargo run --example simple
   macos:
-    runs-on: macos-12
     timeout-minutes: 30
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-14" ]
+    runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
         working-directory: rust
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
             fetch-depth: 0
             lfs: true
@@ -65,11 +108,12 @@ jobs:
       - name: Build
         run: cargo build --all-features
       - name: Run tests
-        run: cargo test --all-features
+        # Run with everything except the integration tests.
+        run: cargo test --features remote,fp16kernels
   windows:
     runs-on: windows-2022
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -87,3 +131,4 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
+  

.github/workflows/update_package_lock_nodejs/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock_nodejs
+description: "Update nodejs's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./nodejs
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
           persist-credentials: false

.github/workflows/update_package_lock_run_nodejs.yml

@@ -0,0 +1,19 @@
+name: Update NodeJs package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock_nodejs
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/upload_wheel/action.yml

@@ -0,0 +1,44 @@
+name: upload-wheel
+
+description: "Upload wheels to Pypi"
+inputs:
+  pypi_token:
+    required: true
+    description: "release token for the repo"
+  fury_token:
+    required: true
+    description: "release token for the fury repo"
+
+runs:
+  using: "composite"
+  steps:
+  - name: Install dependencies
+    shell: bash
+    run: |
+      python -m pip install --upgrade pip
+      pip install twine
+  - name: Choose repo
+    shell: bash
+    id: choose_repo
+    run: |
+      if [ ${{ github.ref }} == "*beta*" ]; then
+        echo "repo=fury" >> $GITHUB_OUTPUT
+      else
+        echo "repo=pypi" >> $GITHUB_OUTPUT
+      fi
+  - name: Publish to PyPI
+    shell: bash
+    env:
+      FURY_TOKEN: ${{ inputs.fury_token }}
+      PYPI_TOKEN: ${{ inputs.pypi_token }}
+    run: |
+      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
+        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
+        echo "Uploading $WHEEL to Fury"
+        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
+      else
+        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
+          --username __token__ \
+          --password $PYPI_TOKEN \
+          target/wheels/lancedb-*.whl
+      fi

.gitignore

@@ -6,7 +6,7 @@
 venv
 
 .vscode
-
+.zed
 rust/target
 rust/Cargo.lock
 
@@ -22,6 +22,11 @@ python/dist
 
 **/.hypothesis
 
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
 ## Javascript
 *.node
 **/node_modules
@@ -29,8 +34,12 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
+nodejs/lancedb/native*
+dist
 
 ## Rust
 target
 
+**/sccache.log
+
 Cargo.lock

.pre-commit-config.yaml

@@ -5,17 +5,17 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
--   repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.277
+    rev: v0.2.2
     hooks:
     - id: ruff
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+- repo: local
   hooks:
-    - id: isort
-      name: isort (python)
+    - id: local-biome-check
+      name: biome check
+      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
+      language: system
+      types: [text]
+      files: "nodejs/.*"
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

Cargo.toml

@@ -1,26 +1,51 @@
 [workspace]
-members = ["rust/ffi/node", "rust/vectordb"]
+members = [
+    "rust/ffi/node",
+    "rust/lancedb",
+    "nodejs",
+    "python",
+    "java/core/lancedb-jni",
+]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
+[workspace.package]
+edition = "2021"
+authors = ["LanceDB Devs <dev@lancedb.com>"]
+license = "Apache-2.0"
+repository = "https://github.com/lancedb/lancedb"
+description = "Serverless, low-latency vector database for AI applications"
+keywords = ["lancedb", "lance", "database", "vector", "search"]
+categories = ["database-implementations"]
+
 [workspace.dependencies]
-lance = { "version" = "=0.8.3", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.3" }
-lance-testing = { "version" = "=0.8.3" }
+lance = { "version" = "=0.13.0", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.13.0" }
+lance-linalg = { "version" = "=0.13.0" }
+lance-testing = { "version" = "=0.13.0" }
+lance-datafusion = { "version" = "=0.13.0" }
 # Note that this one does not include pyarrow
-arrow = { version = "43.0.0", optional = false }
-arrow-array = "43.0"
-arrow-data = "43.0"
-arrow-ipc = "43.0"
-arrow-ord = "43.0"
-arrow-schema = "43.0"
-arrow-arith = "43.0"
-arrow-cast = "43.0"
-half = { "version" = "=2.2.1", default-features = false, features = [
-    "num-traits"
+arrow = { version = "51.0", optional = false }
+arrow-array = "51.0"
+arrow-data = "51.0"
+arrow-ipc = "51.0"
+arrow-ord = "51.0"
+arrow-schema = "51.0"
+arrow-arith = "51.0"
+arrow-cast = "51.0"
+async-trait = "0"
+chrono = "0.4.35"
+datafusion-physical-plan = "37.1"
+half = { "version" = "=2.4.1", default-features = false, features = [
+    "num-traits",
 ] }
+futures = "0"
 log = "0.4"
-object_store = "0.6.1"
+object_store = "0.9.0"
+pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
+num-traits = "0.2"
+regex = "1.10"
+lazy_static = "1"

README.md

@@ -1,14 +1,15 @@
 <div align="center">
 <p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://user-images.githubusercontent.com/917119/226205734-6063d87a-1ecc-45fe-85be-1dea6383a3d8.png">
+<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
 
-**Developer-friendly, serverless vector database for AI applications**
+**Developer-friendly, database for multimodal AI**
 
-<a href="https://lancedb.github.io/lancedb/">Documentation</a> •
-<a href="https://blog.lancedb.com/">Blog</a> •
-<a href="https://discord.gg/zMM32dvNtd">Discord</a> •
-<a href="https://twitter.com/lancedb">Twitter</a>
+<a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+<a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
+[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/) 
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
+[![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
 </p>
 
@@ -19,7 +20,7 @@
 
 <hr />
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -35,7 +36,7 @@ The key features of LanceDB include:
 
 * GPU support in building vector index(*).
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -50,13 +51,19 @@ npm install vectordb
 const lancedb = require('vectordb');
 const db = await lancedb.connect('data/sample-lancedb');
 
-const table = await db.createTable('vectors',
-      [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-       { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
+const table = await db.createTable({
+  name: 'vectors',
+  data:  [
+    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
+  ]
+})
 
-const query = table.search([0.1, 0.3]);
-query.limit = 20;
+const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
+
+// You can also search for rows by specific criteria without involving a vector search.
+const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
 ```
 
 **Python**
@@ -72,9 +79,9 @@ db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
                                {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-result = table.search([100, 100]).limit(2).to_df()
+result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
+* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
 * 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -13,7 +13,9 @@ docker build \
     .
 popd
 
+# We turn on memory swap to avoid OOM killer
 docker run \
     -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
     lancedb-node-manylinux \
     bash ci/manylinux_node/build.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_nodejs
+docker build \
+    -t lancedb-nodejs-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+# We turn on memory swap to avoid OOM killer
+docker run \
+    -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
+    lancedb-nodejs-manylinux \
+    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts.sh

@@ -1,6 +1,7 @@
 # Builds the macOS artifacts (node binaries).
 # Usage: ./ci/build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
 
 prebuild_rust() {
     # Building here for the sake of easier debugging.

ci/build_macos_artifacts_nodejs.sh

@@ -0,0 +1,34 @@
+# Builds the macOS artifacts (nodejs binaries).
+# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+set -e
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/lancedb
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd nodejs
+    echo "Building nodejs library for $1"
+    export RUST_TARGET=$1
+    npm run build-release
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts_nodejs.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (nodejs binaries).
+# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/lancedb"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "nodejs"
+    Write-Host "Building nodejs library for $target"
+    $env:RUST_TARGET=$target
+    npm run build-release
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/bump_version.sh

@@ -0,0 +1,51 @@
+set -e
+
+RELEASE_TYPE=${1:-"stable"}
+BUMP_MINOR=${2:-false}
+TAG_PREFIX=${3:-"v"} # Such as "python-v"
+HEAD_SHA=${4:-$(git rev-parse HEAD)}
+
+readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
+echo "Found previous tag $PREV_TAG"
+
+# Initially, we don't want to tag if we are doing stable, because we will bump
+# again later. See comment at end for why.
+if [[ "$RELEASE_TYPE" == 'stable' ]]; then 
+  BUMP_ARGS="--no-tag"
+fi
+
+# If last is stable and not bumping minor
+if [[ $PREV_TAG != *beta* ]]; then
+    if [[ "$BUMP_MINOR" != "false" ]]; then
+      # X.Y.Z -> X.(Y+1).0-beta.0
+      bump-my-version bump -vv $BUMP_ARGS minor
+    else
+      # X.Y.Z -> X.Y.(Z+1)-beta.0
+      bump-my-version bump -vv $BUMP_ARGS patch
+    fi
+else
+  if [[ "$BUMP_MINOR" != "false" ]]; then
+    # X.Y.Z-beta.N -> X.(Y+1).0-beta.0
+    bump-my-version bump -vv $BUMP_ARGS minor
+  else
+    # X.Y.Z-beta.N -> X.Y.Z-beta.(N+1)
+    bump-my-version bump -vv $BUMP_ARGS pre_n
+  fi
+fi
+
+# The above bump will always bump to a pre-release version. If we are releasing
+# a stable version, bump the pre-release level ("pre_l") to make it stable.
+if [[ $RELEASE_TYPE == 'stable' ]]; then
+  # X.Y.Z-beta.N -> X.Y.Z
+  bump-my-version bump -vv pre_l
+fi
+
+# Validate that we have incremented version appropriately for breaking changes
+NEW_TAG=$(git describe --tags --exact-match HEAD)
+NEW_VERSION=$(echo $NEW_TAG | sed "s/^$TAG_PREFIX//")
+LAST_STABLE_RELEASE=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | grep -v beta | grep -vF "$NEW_TAG" | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
+LAST_STABLE_VERSION=$(echo $LAST_STABLE_RELEASE | sed "s/^$TAG_PREFIX//")
+
+python $SELF_DIR/check_breaking_changes.py $LAST_STABLE_RELEASE $HEAD_SHA $LAST_STABLE_VERSION $NEW_VERSION

ci/check_breaking_changes.py

@@ -0,0 +1,35 @@
+"""
+Check whether there are any breaking changes in the PRs between the base and head commits.
+If there are, assert that we have incremented the minor version.
+"""
+import argparse
+import os
+from packaging.version import parse
+
+from github import Github
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base")
+    parser.add_argument("head")
+    parser.add_argument("last_stable_version")
+    parser.add_argument("current_version")
+    args = parser.parse_args()
+
+    repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"])
+    commits = repo.compare(args.base, args.head).commits
+    prs = (pr for commit in commits for pr in commit.get_pulls())
+
+    for pr in prs:
+        if any(label.name == "breaking-change" for label in pr.labels):
+            print(f"Breaking change in PR: {pr.html_url}")
+            break
+    else:
+        print("No breaking changes found.")
+        exit(0)
+    
+    last_stable_version = parse(args.last_stable_version)
+    current_version = parse(args.current_version)
+    if current_version.minor <= last_stable_version.minor:
+        print("Minor version is not greater than the last stable version.")
+        exit(1)

ci/manylinux_nodejs/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd nodejs
+npm ci
+npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

ci/semver_sort.py

@@ -0,0 +1,35 @@
+"""
+Takes a list of semver strings and sorts them in ascending order.
+"""
+
+import sys
+from packaging.version import parse, InvalidVersion
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("prefix", default="v")
+    args = parser.parse_args()
+
+    # Read the input from stdin
+    lines = sys.stdin.readlines()
+
+    # Parse the versions
+    versions = []
+    for line in lines:
+        line = line.strip()
+        try:
+            version_str = line.removeprefix(args.prefix)
+            version = parse(version_str)
+        except InvalidVersion:
+            # There are old tags that don't follow the semver format
+            print(f"Invalid version: {line}", file=sys.stderr)
+            continue
+        versions.append((line, version))
+
+    # Sort the versions
+    versions.sort(key=lambda x: x[1])
+
+    # Print the sorted versions as original strings
+    for line, _ in versions:
+        print(line)

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:0.14
+    image: localstack/localstack:3.3
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb
+      - SERVICES=s3,dynamodb,kms
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
+      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
       interval: 5s
       retries: 3
       start_period: 10s

dockerfiles/Dockerfile

@@ -0,0 +1,27 @@
+#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
+#Usage docker build -t lancedb:latest -f Dockerfile .
+FROM python:3.10-slim-buster
+
+# Install Rust
+RUN apt-get update && apt-get install -y curl build-essential && \
+  curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Set the environment variable for Rust
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install protobuf compiler
+RUN apt-get install -y protobuf-compiler && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN apt-get -y update &&\
+  apt-get -y upgrade && \
+  apt-get -y install git
+
+
+# Verify installations
+RUN python --version && \
+  rustc --version && \
+  protoc --version
+
+RUN pip install tantivy lancedb

docs/README.md

@@ -0,0 +1,44 @@
+# LanceDB Documentation
+
+LanceDB docs are deployed to https://lancedb.github.io/lancedb/.
+
+Docs is built and deployed automatically by [Github Actions](.github/workflows/docs.yml)
+whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
+unreleased features.
+
+## Building the docs
+
+### Setup
+1. Install LanceDB. From LanceDB repo root: `pip install -e python`
+2. Install dependencies. From LanceDB repo root: `pip install -r docs/requirements.txt`
+3. Make sure you have node and npm setup
+4. Make sure protobuf and libssl are installed
+
+### Building node module and create markdown files
+
+See [Javascript docs README](./src/javascript/README.md)
+
+### Build docs
+From LanceDB repo root:
+
+Run: `PYTHONPATH=. mkdocs build -f docs/mkdocs.yml`
+
+If successful, you should see a `docs/site` directory that you can verify locally.
+
+### Run local server
+
+You can run a local server to test the docs prior to deployment by navigating to the `docs` directory and running the following command:
+
+```bash
+cd docs
+mkdocs serve
+```
+
+### Run doctest for typescript example
+
+```bash
+cd lancedb/docs
+npm i
+npm run build
+npm run all
+```

docs/mkdocs.yml

@@ -1,4 +1,5 @@
-site_name: LanceDB Docs
+site_name: LanceDB
+site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
@@ -8,19 +9,30 @@ theme:
   name: "material"
   logo: assets/logo.png
   favicon: assets/logo.png
+  palette:
+    # Palette toggle for light mode
+    - scheme: lancedb
+      primary: custom
+      toggle:
+        icon: material/weather-night
+        name: Switch to dark mode
+    # Palette toggle for dark mode
+    - scheme: slate
+      primary: custom
+      toggle:
+        icon: material/weather-sunny
+        name: Switch to light mode
   features:
     - content.code.copy
     - content.tabs.link
     - content.action.edit
     - toc.follow
-    - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
     - navigation.footer
     - navigation.tracking
     - navigation.instant
-    - navigation.indexes
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -32,11 +44,10 @@ plugins:
       handlers:
         python:
           paths: [../python]
-        selection:
+          options:
             docstring_style: numpy
-        rendering:
-          heading_level: 4
-          show_source: false
+            heading_level: 3
+            show_source: true
             show_symbol_type_in_heading: true
             show_signature_annotations: true
             show_root_heading: true
@@ -50,91 +61,189 @@ plugins:
 markdown_extensions:
   - admonition
   - footnotes
-- pymdownx.superfences
   - pymdownx.details
   - pymdownx.highlight:
       anchor_linenums: true
       line_spans: __span
       pygments_lang_class: true
   - pymdownx.inlinehilite
-- pymdownx.snippets
+  - pymdownx.snippets:
+      base_path: ..
+      dedent_subsections: true
   - pymdownx.superfences
   - pymdownx.tabbed:
       alternate_style: true
   - md_in_html
+  - attr_list
 
 nav:
   - Home:
-  - 🏢 Home: index.md
-  - 💡 Basics: basic.md
-  - 📚 Guides:
-    - Tables: guides/tables.md
+      - LanceDB: index.md
+      - 🏃🏼‍♂️ Quick start: basic.md
+      - 📚 Concepts:
+          - Vector search: concepts/vector_search.md
+          - Indexing: concepts/index_ivfpq.md
+          - Storage: concepts/storage.md
+          - Data management: concepts/data_management.md
+      - 🔨 Guides:
+          - Working with tables: guides/tables.md
+          - Building an ANN index: ann_indexes.md
           - Vector Search: search.md
-    - SQL filters: sql.md
-    - Indexing: ann_indexes.md
-  - 🧬 Embeddings: embedding.md
-  - 🔍 Python full-text search: fts.md
+          - Full-text search: fts.md
+          - Hybrid search:
+              - Overview: hybrid_search/hybrid_search.md
+              - Comparing Rerankers: hybrid_search/eval.md
+              - Airbnb financial data example: notebooks/hybrid_search.ipynb
+          - Reranking:
+              - Quickstart: reranking/index.md
+              - Cohere Reranker: reranking/cohere.md
+              - Linear Combination Reranker: reranking/linear_combination.md
+              - Cross Encoder Reranker: reranking/cross_encoder.md
+              - ColBERT Reranker: reranking/colbert.md
+              - OpenAI Reranker: reranking/openai.md
+              - Building Custom Rerankers: reranking/custom_reranker.md
+          - Filtering: sql.md
+          - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+          - Configuring Storage: guides/storage.md
+          - Sync -> Async Migration Guide: migration.md
+          - Tuning retrieval performance:
+              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+              - Reranking: guides/tuning_retrievers/2_reranking.md
+      - 🧬 Managing embeddings:
+          - Overview: embeddings/index.md
+          - Embedding functions: embeddings/embedding_functions.md
+          - Available models: embeddings/default_embedding_functions.md
+          - User-defined embedding functions: embeddings/custom_embedding_function.md
+          - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
+          - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
       - 🔌 Integrations:
-    - integrations/index.md
-    - Pandas and PyArrow: python/arrow.md
+          - Tools and data formats: integrations/index.md
+          - Pandas and PyArrow: python/pandas_and_pyarrow.md
+          - Polars: python/polars_arrow.md
           - DuckDB: python/duckdb.md
-    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+          - LangChain:
+            - LangChain 🔗: integrations/langchain.md
+            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+          - LlamaIndex 🦙:
+            - LlamaIndex docs: integrations/llamaIndex.md
+            - LlamaIndex demo:  https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
           - Pydantic: python/pydantic.md
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
-  - 🐍 Python examples:
-    - examples/index.md
+      - 🎯 Examples:
+          - Overview: examples/index.md
+          - 🐍 Python:
+              - Overview: examples/examples_python.md
               - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
               - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
               - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+              - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
               - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
               - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-  - 🌐 Javascript examples:
-    - Examples: examples/index_js.md
+          - 👾 JavaScript:
+              - Overview: examples/examples_js.md
               - Serverless Website Chatbot: examples/serverless_website_chatbot.md
               - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
               - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-- Basics: basic.md
+          - 🦀 Rust:
+              - Overview: examples/examples_rust.md
+      - 💭 FAQs: faq.md
+      - ⚙️ API reference:
+          - 🐍 Python: python/python.md
+          - 👾 JavaScript (vectordb): javascript/modules.md
+          - 👾 JavaScript (lancedb): javascript/modules.md
+          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
+      - ☁️ LanceDB Cloud:
+          - Overview: cloud/index.md
+          - API reference:
+              - 🐍 Python: python/saas-python.md
+              - 👾 JavaScript: javascript/modules.md
+
+  - Quick start: basic.md
+  - Concepts:
+      - Vector search: concepts/vector_search.md
+      - Indexing: concepts/index_ivfpq.md
+      - Storage: concepts/storage.md
+      - Data management: concepts/data_management.md
   - Guides:
-  - Tables: guides/tables.md
+      - Working with tables: guides/tables.md
+      - Building an ANN index: ann_indexes.md
       - Vector Search: search.md
-  - SQL filters: sql.md
-  - Indexing: ann_indexes.md
-- Embeddings: embedding.md
-- Python full-text search: fts.md
+      - Full-text search: fts.md
+      - Hybrid search:
+          - Overview: hybrid_search/hybrid_search.md
+          - Comparing Rerankers: hybrid_search/eval.md
+          - Airbnb financial data example: notebooks/hybrid_search.ipynb
+      - Reranking:
+          - Quickstart: reranking/index.md
+          - Cohere Reranker: reranking/cohere.md
+          - Linear Combination Reranker: reranking/linear_combination.md
+          - Cross Encoder Reranker: reranking/cross_encoder.md
+          - ColBERT Reranker: reranking/colbert.md
+          - OpenAI Reranker: reranking/openai.md
+          - Building Custom Rerankers: reranking/custom_reranker.md
+      - Filtering: sql.md
+      - Versioning & Reproducibility: notebooks/reproducibility.ipynb
+      - Configuring Storage: guides/storage.md
+      - Sync -> Async Migration Guide: migration.md
+      - Tuning retrieval performance:
+          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+          - Reranking: guides/tuning_retrievers/2_reranking.md
+  - Managing Embeddings:
+      - Overview: embeddings/index.md
+      - Embedding functions: embeddings/embedding_functions.md
+      - Available models: embeddings/default_embedding_functions.md
+      - User-defined embedding functions: embeddings/custom_embedding_function.md
+      - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
+      - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - Integrations:
-  - integrations/index.md
-  - Pandas and PyArrow: python/arrow.md
+      - Overview: integrations/index.md
+      - Pandas and PyArrow: python/pandas_and_pyarrow.md
+      - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
-  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-  - LangChain JS/TS 🦜️🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
+      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+      - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
-- Python examples:
+  - Examples:
       - examples/index.md
       - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
       - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
       - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
       - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
       - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-- Javascript examples:
-  - examples/index_js.md
-  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
       - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-- API references:
-  - Python API: python/python.md
-  - Javascript API: javascript/modules.md
-- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
+  - API reference:
+      - Overview: api_reference.md
+      - Python: python/python.md
+      - Javascript (vectordb): javascript/modules.md
+      - Javascript (lancedb): js/modules.md
+      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
+  - LanceDB Cloud:
+      - Overview: cloud/index.md
+      - API reference:
+          - 🐍 Python: python/saas-python.md
+          - 👾 JavaScript: javascript/modules.md
 
 extra_css:
   - styles/global.css
+  - styles/extra.css
+
+extra_javascript:
+  - "extra_js/init_ask_ai_widget.js"
 
 extra:
   analytics:
     provider: google
     property: G-B7NFM40W74
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/lancedb/lancedb
+    - icon: fontawesome/brands/x-twitter
+      link: https://twitter.com/lancedb
+    - icon: fontawesome/brands/linkedin
+      link: https://www.linkedin.com/company/lancedb

docs/package-lock.json

@@ -0,0 +1,132 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-docs-test",
+      "version": "1.0.0",
+      "license": "Apache 2",
+      "dependencies": {
+        "apache-arrow": "file:../node/node_modules/apache-arrow",
+        "vectordb": "file:../node"
+      },
+      "devDependencies": {
+        "@types/node": "^20.11.8",
+        "typescript": "^5.3.3"
+      }
+    },
+    "../node": {
+      "name": "vectordb",
+      "version": "0.4.6",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "@apache-arrow/ts": "^14.0.2",
+        "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^14.0.2",
+        "axios": "^1.4.0"
+      },
+      "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
+        "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
+        "@types/mocha": "^10.0.1",
+        "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
+        "@types/temp": "^0.9.1",
+        "@types/uuid": "^9.0.3",
+        "@typescript-eslint/eslint-plugin": "^5.59.1",
+        "cargo-cp-artifact": "^0.1",
+        "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
+        "eslint": "^8.39.0",
+        "eslint-config-standard-with-typescript": "^34.0.1",
+        "eslint-plugin-import": "^2.26.0",
+        "eslint-plugin-n": "^15.7.0",
+        "eslint-plugin-promise": "^6.1.1",
+        "mocha": "^10.2.0",
+        "openai": "^4.24.1",
+        "sinon": "^15.1.0",
+        "temp": "^0.9.4",
+        "ts-node": "^10.9.1",
+        "ts-node-dev": "^2.0.0",
+        "typedoc": "^0.24.7",
+        "typedoc-plugin-markdown": "^3.15.3",
+        "typescript": "*",
+        "uuid": "^9.0.0"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.4.6",
+        "@lancedb/vectordb-darwin-x64": "0.4.6",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
+      }
+    },
+    "../node/node_modules/apache-arrow": {
+      "version": "14.0.2",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@types/command-line-args": "5.2.0",
+        "@types/command-line-usage": "5.0.2",
+        "@types/node": "20.3.0",
+        "@types/pad-left": "2.1.1",
+        "command-line-args": "5.2.1",
+        "command-line-usage": "7.0.1",
+        "flatbuffers": "23.5.26",
+        "json-bignum": "^0.0.3",
+        "pad-left": "^2.1.0",
+        "tslib": "^2.5.3"
+      },
+      "bin": {
+        "arrow2csv": "bin/arrow2csv.js"
+      }
+    },
+    "node_modules/@types/node": {
+      "version": "20.11.8",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
+      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
+      "dev": true,
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
+    },
+    "node_modules/apache-arrow": {
+      "resolved": "../node/node_modules/apache-arrow",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.3.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
+      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
+      "dev": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
+    "node_modules/vectordb": {
+      "resolved": "../node",
+      "link": true
+    }
+  }
+}

docs/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "auto-generated tests from doc",
+  "author": "dev@lancedb.com",
+  "license": "Apache 2",
+  "dependencies": {
+    "apache-arrow": "file:../node/node_modules/apache-arrow",
+    "vectordb": "file:../node"
+  },
+  "scripts": {
+    "build": "tsc -b && cd ../node && npm run build-release",
+    "example": "npm run build && node",
+    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.8",
+    "typescript": "^5.3.3"
+  }
+}

docs/requirements.txt

@@ -1,4 +1,5 @@
-mkdocs==1.4.2
+mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
-mkdocs-material==9.1.3
+mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
+pydantic

docs/src/ann_indexes.md

@@ -1,29 +1,24 @@
-# ANN (Approximate Nearest Neighbor) Indexes
+# Approximate Nearest Neighbor (ANN) Indexes
 
-You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
+An ANN or a vector index is a data structure specifically designed to efficiently organize and
+search vector data based on their similarity via the chosen distance metric.
+By constructing a vector index, the search space is effectively narrowed down, avoiding the need
+for brute-force scanning of the entire vector space.
+A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-Currently, LanceDB does *not* automatically create the ANN index.
-LanceDB has optimized code for KNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
-If you can live with < 100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+## Disk-based Index
 
-In the future we will look to automatically create and configure the ANN index.
-
-## Types of Index
-
-Lance can support multiple index types, the most widely used one is `IVF_PQ`.
-
-* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
-   and then use **Product Quantization** to compress vectors in each partition.
-* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
-   represent the nearest neighbors of each vector.
+Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
+the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
+See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
 
 ## Creating an IVF_PQ Index
 
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
+
     Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
     ```python
@@ -43,25 +38,42 @@ Lance supports `IVF_PQ` index type by default.
     tbl.create_index(num_partitions=256, num_sub_vectors=96)
     ```
 
-=== "Javascript"
-     ```javascript
-     const vectordb = require('vectordb')
-     const db = await vectordb.connect('data/sample-lancedb')
+=== "Typescript"
 
-     let data = []
-     for (let i = 0; i < 10_000; i++) {
-         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-     }
-     const table = await db.createTable('my_vectors', data)
-     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+    ```typescript
+    --8<--- "docs/src/ann_indexes.ts:import"
+
+    --8<-- "docs/src/ann_indexes.ts:ingest"
     ```
 
-- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
+    ```
+
+    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
+
+The following IVF_PQ paramters can be specified:
+
+- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
   We also support "cosine" and "dot" distance as well.
-- **num_partitions** (default: 256): The number of partitions of the index.
-- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-a single PQ code.
+- **num_partitions**: The number of partitions in the index. The default is the square root
+  of the number of rows.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
+
+- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
+  a single PQ code. The default is the dimension of the vector divided by 16.
+
+!!! note
+
+    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
+    changed in the asynchronous python SDK and node's `lancedb`.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -71,9 +83,40 @@ a single PQ code.
 ### Use GPU to build vector index
 
 Lance Python SDK has experimental GPU support for creating IVF index.
+Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
+
 You can specify the GPU device to train IVF partitions via
 
-- **accelerator**: Specify to `"cuda"`` to enable GPU training.
+- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
+
+=== "Linux"
+
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
+
+=== "MacOS"
+
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
+Troubleshooting:
+
+If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
+PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
 
 ## Querying an ANN Index
 
@@ -92,47 +135,51 @@ There are a couple of parameters that can be used to fine-tune the search:
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
+
     ```python
     tbl.search(np.random.random((1536))) \
         .limit(2) \
         .nprobes(20) \
         .refine_factor(10) \
-         .to_df()
-     ```
+        .to_pandas()
     ```
+
+    ```text
                                               vector       item       _distance
     0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
     ```
 
-=== "Javascript"
-     ```javascript
-     const results_1 = await table
-         .search(Array(1536).fill(1.2))
-         .limit(2)
-         .nprobes(20)
-         .refineFactor(10)
-         .execute()
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/ann_indexes.ts:search1"
     ```
 
-The search will return the data requested in addition to the distance of each item.
+=== "Rust"
 
+    ```rust
+    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
+    ```
+
+    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
+
+The search will return the data requested in addition to the distance of each item.
 
 ### Filtering (where clause)
 
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
+
      ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
-=== "Javascript"
+=== "Typescript"
+
      ```javascript
-     const results_2 = await table
-         .search(Array(1536).fill(1.2))
-         .where("id != '1141'")
-         .execute()
+     --8<-- "docs/src/ann_indexes.ts:search2"
      ```
 
 ### Projections (select clause)
@@ -140,33 +187,42 @@ You can further filter the elements returned by a search using a where clause.
 You can select the columns returned by the query using a select clause.
 
 === "Python"
+
      ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
-     ```
+     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
+
+
+     ```text
                                                    vector _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
      ```
 
-=== "Javascript"
-     ```javascript
-     const results_3 = await table
-         .search(Array(1536).fill(1.2))
-         .select(["id"])
-         .execute()
+=== "Typescript"
+
+     ```typescript
+     --8<-- "docs/src/ann_indexes.ts:search3"
      ```
 
 ## FAQ
 
+### Why do I need to manually create an index?
+
+Currently, LanceDB does _not_ automatically create the ANN index.
+LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
+datasets of the order of ~100K vectors don't require index creation. If you can live with up to
+100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
+
 ### When is it necessary to create an ANN vector index?
 
-`LanceDB` has manually-tuned SIMD code for computing vector distances.
-In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
-For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
+`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
+In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
+We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
+vector indices are usually not necessary.
 
-For large-scale or higher dimension vectors, it is beneficial to create vector index.
+For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
 
 ### How big is my index, and how many memory will it take?
 

docs/src/ann_indexes.ts

@@ -0,0 +1,53 @@
+// --8<-- [start:import]
+import * as vectordb from "vectordb";
+// --8<-- [end:import]
+
+(async () => {
+  // --8<-- [start:ingest]
+  const db = await vectordb.connect("data/sample-lancedb");
+
+  let data = [];
+  for (let i = 0; i < 10_000; i++) {
+    data.push({
+      vector: Array(1536).fill(i),
+      id: `${i}`,
+      content: "",
+      longId: `${i}`,
+    });
+  }
+  const table = await db.createTable("my_vectors", data);
+  await table.createIndex({
+    type: "ivf_pq",
+    column: "vector",
+    num_partitions: 16,
+    num_sub_vectors: 48,
+  });
+  // --8<-- [end:ingest]
+
+  // --8<-- [start:search1]
+  const results_1 = await table
+    .search(Array(1536).fill(1.2))
+    .limit(2)
+    .nprobes(20)
+    .refineFactor(10)
+    .execute();
+  // --8<-- [end:search1]
+
+  // --8<-- [start:search2]
+  const results_2 = await table
+    .search(Array(1536).fill(1.2))
+    .where("id != '1141'")
+    .limit(2)
+    .execute();
+  // --8<-- [end:search2]
+
+  // --8<-- [start:search3]
+  const results_3 = await table
+    .search(Array(1536).fill(1.2))
+    .select(["id"])
+    .limit(2)
+    .execute();
+  // --8<-- [end:search3]
+
+  console.log("Ann indexes: done");
+})();

docs/src/api_reference.md

@@ -0,0 +1,8 @@
+# API Reference
+
+The API reference for the LanceDB client SDKs are available at the following locations:
+
+- [Python](python/python.md)
+- [JavaScript (legacy vectordb package)](javascript/modules.md)
+- [JavaScript (newer @lancedb/lancedb package)](js/modules.md)
+- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -1,55 +1,144 @@
-# Basic LanceDB Functionality
+# Quick start
 
-We'll cover the basics of using LanceDB on your local machine in this section.
+!!! info "LanceDB can be run in a number of ways:"
 
-??? info "LanceDB runs embedded on your backend application, so there is no need to run a separate server."
+    * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
+    * Directly from a client application like a Jupyter notebook for analytical workloads
+    * Deployed as a remote serverless database
 
-      <img src="../assets/lancedb_embedded_explanation.png" width="650px" />
+![](assets/lancedb_embedded_explanation.png)
 
 ## Installation
 
 === "Python"
+
       ```shell
       pip install lancedb
       ```
 
-=== "Javascript"
+=== "Typescript"
+
       ```shell
       npm install vectordb
       ```
 
-## How to connect to a database
+=== "Rust"
+
+    ```shell
+    cargo add lancedb
+    ```
+
+    !!! info "To use the lancedb create, you first need to install protobuf."
+
+    === "macOS"
+
+        ```shell
+        brew install protobuf
+        ```
+
+    === "Ubuntu/Debian"
+
+        ```shell
+        sudo apt install -y protobuf-compiler libssl-dev
+        ```
+
+    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
+
+### Preview releases
+
+Stable releases are created about every 2 weeks. For the latest features and bug
+fixes, you can install the preview release. These releases receive the same
+level of testing as stable releases, but are not guaranteed to be available for
+more than 6 months after they are released. Once your application is stable, we
+recommend switching to stable releases.
 
 === "Python"
+
+      ```shell
+      pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
+      ```
+
+=== "Typescript"
+
+      ```shell
+      npm install vectordb@preview
+      ```
+
+=== "Rust"
+    
+    We don't push preview releases to crates.io, but you can referent the tag
+    in GitHub within your Cargo dependencies:
+
+    ```toml
+    [dependencies]
+    lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" }
+    ```
+
+## Connect to a database
+
+=== "Python"
+
     ```python
-      import lancedb
-      uri = "data/sample-lancedb"
-      db = lancedb.connect(uri)
+    --8<-- "python/python/tests/docs/test_basic.py:imports"
+    --8<-- "python/python/tests/docs/test_basic.py:connect"
+
+    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
     ```
 
-      LanceDB will create the directory if it doesn't exist (including parent directories).
+    !!! note "Asynchronous Python API"
 
-      If you need a reminder of the uri, use the `db.uri` property.
+        The asynchronous Python API is new and has some slight differences compared
+        to the synchronous API.  Feel free to start using the asynchronous version.
+        Once all features have migrated we will start to move the synchronous API to
+        use the same syntax as the asynchronous API.  To help with this migration we
+        have created a [migration guide](migration.md) detailing the differences.
 
-=== "Javascript"
-      ```javascript
-      const lancedb = require("vectordb");
+=== "Typescript"
 
-      const uri = "data/sample-lancedb";
-      const db = await lancedb.connect(uri);
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:import"
+
+    --8<-- "docs/src/basic_legacy.ts:open_db"
     ```
 
+    !!! note "`@lancedb/lancedb` vs. `vectordb`"
+
+        The Javascript SDK was originally released as `vectordb`.  In an effort to
+        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
+        API is being released as `lancedb`.  If you are starting new work we encourage
+        you to try out `lancedb`.  Once the new API is feature complete we will begin
+        slowly deprecating `vectordb` in favor of `lancedb`. There is a
+        [migration guide](migration.md) detailing the differences which will assist
+        you in this process.
+
+=== "Rust"
+
+    ```rust
+    #[tokio::main]
+    async fn main() -> Result<()> {
+        --8<-- "rust/lancedb/examples/simple.rs:connect"
+    }
+    ```
+
+    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
+
 LanceDB will create the directory if it doesn't exist (including parent directories).
 
 If you need a reminder of the uri, you can call `db.uri()`.
 
-## How to create a table
+## Create a table
+
+### Create a table from initial data
+
+If you have data to insert into the table at creation time, you can simultaneously create a
+table and insert the data into it. The schema of the data will be used as the schema of the
+table.
 
 === "Python"
+
     ```python
-      tbl = db.create_table("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    --8<-- "python/python/tests/docs/test_basic.py:create_table"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
     ```
 
     If the table already exists, LanceDB will raise an error by default.
@@ -57,125 +146,233 @@ We'll cover the basics of using LanceDB on your local machine in this section.
     to the `create_table` method.
 
     You can also pass in a pandas DataFrame directly:
+
     ```python
-      import pandas as pd
-      df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      tbl = db.create_table("table_from_df", data=df)
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
-=== "Javascript"
-      ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
-      ```
+=== "Typescript"
 
-!!! warning
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_table"
+    ```
 
     If the table already exists, LanceDB will raise an error by default.
     If you want to overwrite the table, you can pass in `mode="overwrite"`
     to the `createTable` function.
 
-??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
+=== "Rust"
 
-### Creating an empty table
-
-Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema.
-
-=== "Python"
-      ```python
-      import pyarrow as pa
-      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
-      tbl = db.create_table("empty_table", schema=schema)
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:create_table"
     ```
 
-## How to open an existing table
+    If the table already exists, LanceDB will raise an error by default.  See
+    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
+    for details on how to overwrite (or open) existing tables instead.
 
-Once created, you can open a table using the following code:
+    !!! Providing table records in Rust
+
+        The Rust SDK currently expects data to be provided as an Arrow
+        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
+        Support for additional formats (such as serde or polars) is on the roadmap.
+
+!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
+
+!!! info "Automatic embedding generation with Embedding API"
+    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
+
+### Create an empty table
+
+Sometimes you may not have the data to insert into the table at creation time.
+In this case, you can create an empty table and specify the schema, so that you can add
+data to the table at a later time (as long as it conforms to the schema). This is
+similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
+
       ```python
-      tbl = db.open_table("my_table")
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+      ```
+
+    !!! note "You can define schema in Pydantic"
+        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
+    ```
+
+## Open an existing table
+
+Once created, you can open a table as follows:
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:open_table"
+    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    const tbl = await db.openTable("myTable");
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
     ```
 
 If you forget the name of your table, you can always get a listing of all table names:
 
+=== "Python"
+
     ```python
-      print(db.table_names())
+    --8<-- "python/python/tests/docs/test_basic.py:table_names"
+    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
 === "Javascript"
-      ```javascript
-      const tbl = await db.openTable("my_table");
-      ```
-
-      If you forget the name of your table, you can always get a listing of all table names:
 
     ```javascript
     console.log(await db.tableNames());
     ```
 
-## How to add data to a table
+=== "Rust"
 
-After a table has been created, you can always add more data to it using
-
-=== "Python"
-      ```python
-
-      # Option 1: Add a list of dicts to a table
-      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
-            {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
-      tbl.add(data)
-
-      # Option 2: Add a pandas DataFrame to a table
-      df = pd.DataFrame(data)
-      tbl.add(data)
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:list_names"
     ```
 
-=== "Javascript"
-      ```javascript
-      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-              {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
-      ```
+## Add data to a table
 
-## How to search for (approximate) nearest neighbors
-
-Once you've embedded the query, you can find its nearest neighbors using the following code:
+After a table has been created, you can always add more data to it as follows:
 
 === "Python"
+
     ```python
-      tbl.search([100, 100]).limit(2).to_df()
+    --8<-- "python/python/tests/docs/test_basic.py:add_data"
+    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+    ```
+
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:add"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:add"
+    ```
+
+## Search for nearest neighbors
+
+Once you've embedded the query, you can find its nearest neighbors as follows:
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
     ```
 
     This returns a pandas DataFrame with the results.
 
-=== "Javascript"
-      ```javascript
-      const query = await tbl.search([100, 100]).limit(2).execute();
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:search"
     ```
 
-## How to delete rows from a table
+=== "Rust"
+
+    ```rust
+    use futures::TryStreamExt;
+
+    --8<-- "rust/lancedb/examples/simple.rs:search"
+    ```
+
+    !!! Query vectors in Rust
+        Rust does not yet support automatic execution of embedding functions.  You will need to
+        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
+        https://github.com/lancedb/lancedb/issues/994
+
+        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
+        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
+
+By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
+For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
+LanceDB allows you to create an ANN index on a table as follows:
+
+=== "Python"
+
+    ```py
+    --8<-- "python/python/tests/docs/test_basic.py:create_index"
+    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
+    ```
+
+=== "Typescript"
+
+    ```{.typescript .ignore}
+    --8<-- "docs/src/basic_legacy.ts:create_index"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:create_index"
+    ```
+
+!!! note "Why do I need to create an index manually?"
+LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
+for really fast retrievals via a disk-based index, and the second is that data and query workloads can
+be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
+to fine-tune index size, query latency and accuracy. See the section on
+[ANN indexes](ann_indexes.md) for more details.
+
+## Delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
 which rows to delete, provide a filter that matches on the metadata columns.
 This can delete any number of rows that match the filter.
 
 === "Python"
+
     ```python
-      tbl.delete('item = "fizz"')
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
-=== "Javascript"
-      ```javascript
-      await tbl.delete('item = "fizz"')
+=== "Typescript"
+
+    ```typescript
+    --8<-- "docs/src/basic_legacy.ts:delete"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:delete"
     ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause on a search. They can be as simple or complex as needed.
-To see what expressions are supported, see the [SQL filters](sql.md) section.
-
+as the `where()` clause (`only_if()` in Rust) on a search. They can be as
+simple or complex as needed. To see what expressions are supported, see the
+[SQL filters](sql.md) section.
 
 === "Python"
 
@@ -185,28 +382,44 @@ To see what expressions are supported, see the [SQL filters](sql.md) section.
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-## How to remove a table
+=== "Rust"
+
+      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
+
+## Drop a table
 
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
+
       ```python
-      db.drop_table("my_table")
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
       By default, if the table does not exist an exception is raised. To suppress this,
       you can pass in `ignore_missing=True`.
 
+=== "Typescript"
 
-## What's next
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
+      ```
 
-This section covered the very basics of the LanceDB API.
-LanceDB supports many additional features when creating indices to speed up search and options for search.
-These are contained in the next section of the documentation.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised.
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
+    ```
+
+!!! note "Bundling `vectordb` apps with Webpack"
+
+    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
 
-## Note: Bundling vectorDB apps with webpack
-Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
     ```javascript
     /** @type {import('next').NextConfig} */
     module.exports = ({
@@ -216,3 +429,22 @@ module.exports = ({
     }
     })
     ```
+
+## Using the Embedding API
+You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
+    ```
+
+Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
+
+
+## What's next
+
+This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
+
+If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.

docs/src/basic_legacy.ts

@@ -0,0 +1,92 @@
+// --8<-- [start:import]
+import * as lancedb from "vectordb";
+import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
+// --8<-- [end:import]
+import * as fs from "fs";
+import { Table as ArrowTable, Utf8 } from "apache-arrow";
+
+const example = async () => {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  // --8<-- [start:open_db]
+  const lancedb = require("vectordb");
+  const uri = "data/sample-lancedb";
+  const db = await lancedb.connect(uri);
+  // --8<-- [end:open_db]
+
+  // --8<-- [start:create_table]
+  const tbl = await db.createTable(
+    "myTable",
+    [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ],
+    { writeMode: lancedb.WriteMode.Overwrite }
+  );
+  // --8<-- [end:create_table]
+
+  // --8<-- [start:add]
+  const newData = Array.from({ length: 500 }, (_, i) => ({
+    vector: [i, i + 1],
+    item: "fizz",
+    price: i * 0.1,
+  }));
+  await tbl.add(newData);
+  // --8<-- [end:add]
+
+  // --8<-- [start:create_index]
+  await tbl.createIndex({
+    type: "ivf_pq",
+    num_partitions: 2,
+    num_sub_vectors: 2,
+  });
+  // --8<-- [end:create_index]
+
+  // --8<-- [start:create_empty_table]
+  const schema = new Schema([
+    new Field("id", new Int32()),
+    new Field("name", new Utf8()),
+  ]);
+  const empty_tbl = await db.createTable({ name: "empty_table", schema });
+  // --8<-- [end:create_empty_table]
+
+  // --8<-- [start:create_f16_table]
+  const dim = 16
+  const total = 10
+  const f16_schema = new Schema([
+      new Field('id', new Int32()),
+      new Field(
+        'vector',
+        new FixedSizeList(dim, new Field('item', new Float16(), true)),
+        false
+      )
+    ])
+  const data = lancedb.makeArrowTable(
+      Array.from(Array(total), (_, i) => ({
+        id: i,
+        vector: Array.from(Array(dim), Math.random)
+      })),
+      { f16_schema }
+    )
+  const table = await db.createTable('f16_tbl', data)
+  // --8<-- [end:create_f16_table]
+
+  // --8<-- [start:search]
+  const query = await tbl.search([100, 100]).limit(2).execute();
+  // --8<-- [end:search]
+  console.log(query);
+
+  // --8<-- [start:delete]
+  await tbl.delete('item = "fizz"');
+  // --8<-- [end:delete]
+
+  // --8<-- [start:drop_table]
+  await db.dropTable("myTable");
+  // --8<-- [end:drop_table]
+};
+
+async function main() {
+  await example();
+  console.log("Basic example: done");
+}
+
+main();

docs/src/cloud/index.md

@@ -0,0 +1,17 @@
+# About LanceDB Cloud
+
+LanceDB Cloud is a SaaS (software-as-a-service) solution that runs serverless in the cloud, clearly separating storage from compute. It's designed to be highly scalable without breaking the bank. LanceDB Cloud is currently in private beta with general availability coming soon, but you can apply for early access with the private beta release by signing up below.
+
+[Try out LanceDB Cloud](https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms){ .md-button .md-button--primary }
+
+## Architecture
+
+LanceDB Cloud provides the same underlying fast vector store that powers the OSS version, but without the need to maintain your own infrastructure. Because it's serverless, you only pay for the storage you use, and you can scale compute up and down as needed depending on the size of your data and its associated index.
+
+![](../assets/lancedb_cloud.png)
+
+## Transitioning from the OSS to the Cloud version
+
+The OSS version of LanceDB is designed to be embedded in your application, and it runs in-process. This makes it incredibly simple to self-host your own AI retrieval workflows for RAG and more and build and test out your concepts on your own infrastructure. The OSS version is forever free, and you can continue to build and integrate LanceDB into your existing backend applications without any added costs.
+
+Should you decide that you need a managed deployment in production, it's possible to seamlessly transition from the OSS to the cloud version by changing the connection string to point to a remote database instead of a local one. With LanceDB Cloud, you can take your AI application from development to production without major code changes or infrastructure burden.

docs/src/concepts/data_management.md

@@ -0,0 +1,62 @@
+# Data management
+
+This section covers concepts related to managing your data over time in LanceDB.
+
+## A primer on Lance
+
+Because LanceDB is built on top of the [Lance](https://lancedb.github.io/lance/) data format, it helps to understand some of its core ideas. Just like Apache Arrow, Lance is a fast columnar data format, but it has the added benefit of being versionable, query and train ML models on. Lance is designed to be used with simple and complex data types, like tabular data, images, videos audio, 3D point clouds (which are deeply nested) and more.
+
+The following concepts are important to keep in mind:
+
+- Data storage is columnar and is interoperable with other columnar formats (such as Parquet) via Arrow
+- Data is divided into fragments that represent a subset of the data
+- Data is versioned, with each insert operation creating a new version of the dataset and an update to the manifest that tracks versions via metadata
+
+!!! note
+    1. First, each version contains metadata and just the new/updated data in your transaction. So if you have 100 versions, they aren't 100 duplicates of the same data. However, they do have 100x the metadata overhead of a single version, which can result in slower queries.  
+    2. Second, these versions exist to keep LanceDB scalable and consistent. We do not immediately blow away old versions when creating new ones because other clients might be in the middle of querying the old version. It's important to retain older versions for as long as they might be queried.
+
+## What are fragments?
+
+Fragments are chunks of data in a Lance dataset. Each fragment includes multiple files that contain several columns in the chunk of data that it represents.
+
+## Compaction
+
+As you insert more data, your dataset will grow and you'll need to perform *compaction* to maintain query throughput (i.e., keep latencies down to a minimum). Compaction is the process of merging fragments together to reduce the amount of metadata that needs to be managed, and to reduce the number of files that need to be opened while scanning the dataset.
+
+### How does compaction improve performance?
+
+Compaction performs the following tasks in the background:
+
+- Removes deleted rows from fragments
+- Removes dropped columns from fragments
+- Merges small fragments into larger ones
+
+Depending on the use case and dataset, optimal compaction will have different requirements. As a rule of thumb:
+
+- It’s always better to use *batch* inserts rather than adding 1 row at a time (to avoid too small fragments). If single-row inserts are unavoidable, run compaction on a regular basis to merge them into larger fragments.
+- Keep the number of fragments under 100, which is suitable for most use cases (for *really* large datasets of >500M rows, more fragments might be needed)
+
+## Deletion
+
+Although Lance allows you to delete rows from a dataset, it does not actually delete the data immediately. It simply marks the row as deleted in the `DataFile` that represents a fragment. For a given version of the dataset, each fragment can have up to one deletion file (if no rows were ever deleted from that fragment, it will not have a deletion file). This is important to keep in mind because it means that the data is still there, and can be recovered if needed, as long as that version still exists based on your backup policy.
+
+## Reindexing
+
+Reindexing is the process of updating the index to account for new data, keeping good performance for queries. This applies to either a full-text search (FTS) index or a vector index. For ANN search, new data will always be included in query results, but queries on tables with unindexed data will fallback to slower search methods for the new parts of the table. This is another important operation to run periodically as your data grows, as it also improves performance. This is especially important if you're appending large amounts of data to an existing dataset.
+
+!!! tip
+    When adding new data to a dataset that has an existing index (either FTS or vector), LanceDB doesn't immediately update the index until a reindex operation is complete.
+
+Both LanceDB OSS and Cloud support reindexing, but the process (at least for now) is different for each, depending on the type of index.
+
+When a reindex job is triggered in the background, the entire data is reindexed, but in the interim as new queries come in, LanceDB will combine results from the existing index with exhaustive kNN search on the new data. This is done to ensure that you're still searching on all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable.
+
+### Vector reindex
+
+* LanceDB Cloud supports incremental reindexing, where a background process will trigger a new index build for you automatically when new data is added to a dataset
+* LanceDB OSS requires you to manually trigger a reindex operation -- we are working on adding incremental reindexing to LanceDB OSS as well
+
+### FTS reindex
+
+FTS reindexing is supported in both LanceDB OSS and Cloud, but requires that it's manually rebuilt once you have a significant enough amount of new data added that needs to be reindexed. We [updated](https://github.com/lancedb/lancedb/pull/762) Tantivy's default heap size from 128MB to 1GB in LanceDB to make it much faster to reindex, by up to 10x from the default settings.

docs/src/concepts/index_ivfpq.md

@@ -0,0 +1,84 @@
+# Understanding LanceDB's IVF-PQ index
+
+An ANN (Approximate Nearest Neighbors) index is a data structure that represents data in a way that makes it more efficient to search and retrieve. Using an ANN index is faster, but less accurate than kNN or brute force search because, in essence, the index is a lossy representation of the data.
+
+LanceDB is fundamentally different from other vector databases in that it is built on top of [Lance](https://github.com/lancedb/lance), an open-source columnar data format designed for performant ML workloads and fast random access. Due to the design of Lance, LanceDB's indexing philosophy adopts a primarily *disk-based* indexing philosophy.
+
+## IVF-PQ
+
+IVF-PQ is a composite index that combines inverted file index (IVF) and product quantization (PQ). The implementation in LanceDB provides several parameters to fine-tune the index's size, query throughput, latency and recall, which are described later in this section.
+
+### Product quantization
+
+Quantization is a compression technique used to reduce the dimensionality of an embedding to speed up search.
+
+Product quantization (PQ) works by dividing a large, high-dimensional vector of size into equally sized subvectors. Each subvector is assigned a "reproduction value" that maps to the nearest centroid of points for that subvector. The reproduction values are then assigned to a codebook using unique IDs, which can be used to reconstruct the original vector.
+
+![](../assets/ivfpq_pq_desc.png)
+
+It's important to remember that quantization is a *lossy process*, i.e., the reconstructed vector is not identical to the original vector. This results in a trade-off between the size of the index and the accuracy of the search results.
+
+As an example, consider starting with 128-dimensional vector consisting of 32-bit floats. Quantizing it to an 8-bit integer vector with 4 dimensions as in the image above, we can significantly reduce memory requirements.
+
+!!! example "Effect of quantization"
+
+    Original: `128 × 32 = 4096` bits
+    Quantized: `4 × 8 = 32` bits
+
+    Quantization results in a **128x** reduction in memory requirements for each vector in the index, which is substantial.
+
+### Inverted file index
+
+While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
+
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+
+![](../assets/ivfpq_ivf_desc.webp)
+
+During query time, depending on where the query lands in vector space, it may be close to the border of multiple Voronoi cells, which could make the top-k results ambiguous and span across multiple cells. To address this, the IVF-PQ introduces the `nprobe` parameter, which controls the number of Voronoi cells to search during a query. The higher the `nprobe`, the more accurate the results, but the slower the query.
+
+![](../assets/ivfpq_query_vector.webp)
+
+## Putting it all together
+
+We can combine the above concepts to understand how to build and query an IVF-PQ index in LanceDB.
+
+### Construct index
+
+There are three key parameters to set when constructing an IVF-PQ index:
+
+* `metric`: Use an `L2` euclidean distance metric. We also support `dot` and `cosine` distance.
+* `num_partitions`: The number of partitions in the IVF portion of the index.
+* `num_sub_vectors`: The number of sub-vectors that will be created during Product Quantization (PQ).
+
+In Python, the index can be created as follows:
+
+```python
+# Create and train the index for a 1536-dimensional vector
+# Make sure you have enough data in the table for an effective training step
+tbl.create_index(metric="L2", num_partitions=256, num_sub_vectors=96)
+```
+
+The `num_partitions` is usually chosen to target a particular number of vectors per partition. `num_sub_vectors` is typically chosen based on the desired recall and the dimensionality of the vector. See the [FAQs](#faq) below for best practices on choosing these parameters.
+
+
+### Query the index
+
+```python
+# Search using a random 1536-dimensional embedding
+tbl.search(np.random.random((1536))) \
+    .limit(2) \
+    .nprobes(20) \
+    .refine_factor(10) \
+    .to_pandas()
+```
+
+The above query will perform a search on the table `tbl` using the given query vector, with the following parameters:
+
+* `limit`: The number of results to return
+* `nprobes`: The number of probes determines the distribution of vector space. While a higher number enhances search accuracy, it also results in slower performance. Typically, setting `nprobes` to cover 5–10% of the dataset proves effective in achieving high recall with minimal latency.
+* `refine_factor`: Refine the results by reading extra elements and re-ranking them in memory. A higher number makes the search more accurate but also slower (see the [FAQ](../faq.md#do-i-need-to-set-a-refine-factor-when-using-an-index) page for more details on this).
+* `to_pandas()`: Convert the results to a pandas DataFrame
+
+And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
+To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.

docs/src/concepts/storage.md

@@ -0,0 +1,80 @@
+# Storage
+
+LanceDB is among the only vector databases built on top of multiple modular components designed from the ground-up to be efficient on disk. This gives it the unique benefit of being flexible enough to support multiple storage backends, including local NVMe, EBS, EFS and many other third-party APIs that connect to the cloud.
+
+It is important to understand the tradeoffs between cost and latency for your specific application and use case. This section will help you understand the tradeoffs between the different storage backends.
+
+## Storage options
+
+We've prepared a simple diagram to showcase the thought process that goes into choosing a storage backend when using LanceDB OSS, Cloud or Enterprise.
+
+![](../assets/lancedb_storage_tradeoffs.png)
+
+When architecting your system, you'd typically ask yourself the following questions to decide on a storage option:
+
+1. **Latency**: How fast do I need results? What do the p50 and also p95 look like?
+2. **Scalability**: Can I scale up the amount of data and QPS easily?
+3. **Cost**: To serve my application, what’s the all-in cost of *both* storage and serving infra?
+4. **Reliability/Availability**: How does replication work? Is disaster recovery addressed?
+
+## Tradeoffs
+
+This section reviews the characteristics of each storage option in four dimensions: latency, scalability, cost and reliability.
+
+**We begin with the lowest cost option, and end with the lowest latency option.**
+
+### 1. S3 / GCS / Azure Blob Storage
+
+!!! tip "Lowest cost, highest latency"
+
+    - **Latency** ⇒ Has the highest latency. p95 latency is also substantially worse than p50. In general you get results in the order of several hundred milliseconds
+    - **Scalability** ⇒ Infinite on storage, however, QPS will be limited by S3 concurrency limits
+    - **Cost** ⇒ Lowest (order of magnitude cheaper than other options)
+    - **Reliability/Availability** ⇒ Highly available, as blob storage like S3 are critical infrastructure that form the backbone of the internet.
+
+Another important point to note is that LanceDB is designed to separate storage from compute, and the underlying Lance format stores the data in numerous immutable fragments. Due to these factors, LanceDB is a great storage option that addresses the _N + 1_ query problem. i.e., when a high query throughput is required, query processes can run in a stateless manner and be scaled up and down as needed.
+
+### 2. EFS / GCS Filestore / Azure File Storage
+
+!!! info "Moderately low cost, moderately low latency (<100ms)"
+
+    - **Latency** ⇒ Much better than object/blob storage but not as good as EBS/Local disk; < 100ms p95 achievable
+    - **Scalability** ⇒ High, but the bottleneck will be the IOPs limit, but when scaling you can provision multiple EFS volumes
+    - **Cost** ⇒ Significantly more expensive than S3 but still very cost effective compared to in-memory dbs. Inactive data in EFS is also automatically tiered to S3-level costs.
+    - **Reliability/Availability** ⇒ Highly available, as query nodes can go down without affecting EFS.  However, EFS does not provide replication / backup - this must be managed manually.
+
+A recommended best practice is to keep a copy of the data on S3 for disaster recovery scenarios. If any downtime is unacceptable, then you would need another EFS with a copy of the data. This is still much cheaper than EC2 instances holding multiple copies of the data.
+
+### 3. Third-party storage solutions
+
+Solutions like [MinIO](https://blog.min.io/lancedb-trusted-steed-against-data-complexity/), WekaFS, etc. that deliver S3 compatible API with much better performance than S3.
+
+!!! info "Moderately low cost, moderately low latency (<100ms)"
+
+    - **Latency** ⇒ Should be similar latency to EFS, better than S3 (<100ms)
+    - **Scalability** ⇒ Up to the solutions architect, who can add as many nodes to their MinIO or other third-party provider's cluster as needed
+    - **Cost** ⇒ Definitely higher than S3. The cost can be marginally higher than EFS until you get to maybe >10TB scale with high utilization
+    - **Reliability/Availability** ⇒ These are all shareable by lots of nodes, quality/cost of replication/backup depends on the vendor
+
+
+### 4. EBS / GCP Persistent Disk / Azure Managed Disk
+
+!!! info "Very low latency (<30ms), higher cost"
+
+    - **Latency** ⇒ Very good, pretty close to local disk. You’re looking at <30ms latency in most cases
+    - **Scalability** ⇒ EBS is not shareable between instances. If deployed via k8s, it can be shared between pods that live on the same instance, but beyond that you would need to shard data or make an additional copy
+    - **Cost** ⇒ Higher than EFS. There are some hidden costs to EBS as well if you’re paying for IO.
+    - **Reliability/Availability** ⇒ Not shareable between instances but can be shared between pods on the same instance. Survives instance termination. No automatic backups.
+
+Just like EFS, an EBS or persistent disk setup requires more manual work to manage data sharding, backups and capacity.
+
+### 5. Local disk (SSD/NVMe)
+
+!!! danger "Lowest latency (<10ms), highest cost"
+
+    - **Latency** ⇒ Lowest latency with modern NVMe drives, <10ms p95
+    - **Scalability** ⇒ Difficult to scale on cloud. Also need additional copies / sharding if QPS needs to be higher
+    - **Cost** ⇒ Highest cost; the main issue with keeping your application and storage tightly integrated is that it’s just not really possible to scale this up in cloud environments
+    - **Reliability/Availability** ⇒ If the instance goes down, so does your data. You have to be _very_ diligent about backing up your data
+
+As a rule of thumb, local disk should be your storage option if you require absolutely *crazy low* latency and you’re willing to do a bunch of data management work to make it happen.

docs/src/concepts/vector_search.md

@@ -0,0 +1,36 @@
+# Vector search
+
+Vector search is a technique used to search for similar items based on their vector representations, called embeddings. It is also known as similarity search, nearest neighbor search, or approximate nearest neighbor search.
+
+Raw data (e.g. text, images, audio, etc.) is converted into embeddings via an embedding model, which are then stored in a vector database like LanceDB. To perform similarity search at scale, an index is created on the stored embeddings, which can then used to perform fast lookups.
+
+![](../assets/vector-db-basics.png)
+
+## Embeddings
+
+Modern machine learning models can be trained to convert raw data into embeddings, represented as arrays (or vectors) of floating point numbers of fixed dimensionality. What makes embeddings useful in practice is that the position of an embedding in vector space captures some of the semantics of the data, depending on the type of model and how it was trained. Points that are close to each other in vector space are considered similar (or appear in similar contexts), and points that are far away are considered dissimilar.
+
+Large datasets of multi-modal data (text, audio, images, etc.) can be converted into embeddings with the appropriate model. Projecting the vectors' principal components in 2D space results in groups of vectors that represent similar concepts clustering together, as shown below.
+
+![](../assets/embedding_intro.png)
+
+## Indexes
+
+Embeddings for a given dataset are made searchable via an **index**. The index is constructed by using data structures that store the embeddings such that it's very efficient to perform scans and lookups on them. A key distinguishing feature of LanceDB is it uses a disk-based index: IVF-PQ, which is a variant of the Inverted File Index (IVF) that uses Product Quantization (PQ) to compress the embeddings.
+
+See the [IVF-PQ](./index_ivfpq.md) page for more details on how it works.
+
+## Brute force search
+
+The simplest way to perform vector search is to perform a brute force search, without an index, where the distance between the query vector and all the vectors in the database are computed, with the top-k closest vectors returned. This is equivalent to a k-nearest neighbours (kNN) search in vector space.
+
+![](../assets/knn_search.png)
+
+As you can imagine, the brute force approach is not scalable for datasets larger than a few hundred thousand vectors, as the latency of the search grows linearly with the size of the dataset. This is where approximate nearest neighbour (ANN) algorithms come in.
+
+## Approximate nearest neighbour (ANN) search
+
+Instead of performing an exhaustive search on the entire database for each and every query, approximate nearest neighbour (ANN) algorithms use an index to narrow down the search space, which significantly reduces query latency. The trade-off is that the results are not guaranteed to be the true nearest neighbors of the query, but are usually "good enough" for most use cases.
+
+
+

docs/src/embedding.md

@@ -1,142 +0,0 @@
-# Embedding Functions
-
-Embeddings are high dimensional floating-point vector representations of your data or query.
-Anything can be embedded using some embedding model or function.
-For a given embedding function, the output will always have the same number of dimensions.
-
-## Creating an embedding function
-
-Any function that takes as input a batch (list) of data and outputs a batch (list) of embeddings
-can be used by LanceDB as an embedding function. The input and output batch sizes should be the same.
-
-### HuggingFace example
-
-One popular free option would be to use the [sentence-transformers](https://www.sbert.net/) library from HuggingFace.
-You can install this using pip: `pip install sentence-transformers`.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-name="paraphrase-albert-small-v2"
-model = SentenceTransformer(name)
-
-# used for both training and querying
-def embed_func(batch):
-    return [model.encode(sentence) for sentence in batch]
-```
-
-Please note that currently HuggingFace is only supported in the Python SDK.
-
-### OpenAI example
-
-You can also use an external API like OpenAI to generate embeddings
-
-=== "Python"
-      ```python
-        import openai
-        import os
-
-        # Configuring the environment variable OPENAI_API_KEY
-        if "OPENAI_API_KEY" not in os.environ:
-        # OR set the key here as a variable
-        openai.api_key = "sk-..."
-
-        # verify that the API key is working
-        assert len(openai.Model.list()["data"]) > 0
-
-        def embed_func(c):
-            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-            return [record["embedding"] for record in rs["data"]]
-      ```
-
-=== "Javascript"
-      ```javascript
-        const lancedb = require("vectordb");
-
-        // You need to provide an OpenAI API key
-        const apiKey = "sk-..."
-        // The embedding function will create embeddings for the 'text' column
-        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-      ```
-
-## Applying an embedding function
-
-=== "Python"
-    Using an embedding function, you can apply it to raw data
-    to generate embeddings for each row.
-
-    Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
-    you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/python/#lancedb.embeddings.with_embeddings)
-    function to generate embeddings and add create a combined pyarrow table:
-
-
-     ```python
-     import pandas as pd
-     from lancedb.embeddings import with_embeddings
-
-     df = pd.DataFrame([{"text": "pepperoni"},
-                        {"text": "pineapple"}])
-     data = with_embeddings(embed_func, df)
-
-     # The output is used to create / append to a table
-     # db.create_table("my_table", data=data)
-     ```
-
-     If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-     By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-     using the `batch_size` parameter to `with_embeddings`.
-
-     LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-     API call is reliable.
-
-=== "Javascript"
-     Using an embedding function, you can apply it to raw data
-     to generate embeddings for each row.
-
-     You can just pass the embedding function created previously and LanceDB will automatically generate
-     embededings for your data.
-
-      ```javascript
-      const db = await lancedb.connect("data/sample-lancedb");
-      const data = [
-        { text: 'pepperoni'  },
-        { text: 'pineapple' }
-      ]
-
-      const table = await db.createTable('vectors', data, embedding)
-      ```
-
-
-## Searching with an embedding function
-
-At inference time, you also need the same embedding function to embed your query text.
-It's important that you use the same model / function otherwise the embedding vectors don't
-belong in the same latent space and your results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     tbl.search(query_vector).limit(10).to_df()
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "Javascript"
-     ```javascript
-      const results = await table
-        .search("What's the best pizza topping?")
-        .limit(10)
-        .execute()
-     ```
-
-     The above snippet returns an array of records with the 10 closest vectors to the query.
-
-
-## Roadmap
-
-In the near future, we'll be integrating the embedding functions deeper into LanceDB<br/>.
-The goal is that you just have to configure the function once when you create the table,
-and then you'll never have to deal with embeddings / vectors after that unless you want to.
-We'll also integrate more popular models and APIs.

docs/src/embeddings/custom_embedding_function.md

@@ -0,0 +1,212 @@
+To use your own custom embedding function, you can follow these 2 simple steps:
+
+1. Create your embedding function by implementing the `EmbeddingFunction` interface
+2. Register your embedding function in the global `EmbeddingFunctionRegistry`.
+
+Let us see how this looks like in action.
+
+![](../assets/embeddings_api.png)
+
+`EmbeddingFunction` and `EmbeddingFunctionRegistry` handle low-level details for serializing schema and model information as metadata. To build a custom embedding function, you don't have to worry about the finer details - simply focus on setting up the model and leave the rest to LanceDB.
+
+## `TextEmbeddingFunction` interface
+
+There is another optional layer of abstraction available: `TextEmbeddingFunction`. You can use this abstraction if your model isn't multi-modal in nature and only needs to operate on text. In such cases, both the source and vector fields will have the same work for vectorization, so you simply just need to setup the model and rest is handled by `TextEmbeddingFunction`. You can read more about the class and its attributes in the class reference.
+
+Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
+
+```python
+from lancedb.embeddings import register
+from lancedb.util import attempt_import_or_raise
+
+@register("sentence-transformers")
+class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+    name: str = "all-MiniLM-L6-v2"
+    # set more default instance vars like device, etc.
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._ndims = None
+    
+    def generate_embeddings(self, texts):
+        return self._embedding_model().encode(list(texts), ...).tolist()
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = len(self.generate_embeddings("foo")[0])
+        return self._ndims
+
+    @cached(cache={}) 
+    def _embedding_model(self):
+        return sentence_transformers.SentenceTransformer(name)
+```
+
+This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.
+
+Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
+
+```python
+from lancedb.pydantic import LanceModel, Vector
+
+registry = EmbeddingFunctionRegistry.get_instance()
+stransformer = registry.get("sentence-transformers").create()
+
+class TextModelSchema(LanceModel):
+    vector: Vector(stransformer.ndims) = stransformer.VectorField()
+    text: str = stransformer.SourceField()
+
+tbl = db.create_table("table", schema=TextModelSchema)
+
+tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
+result = tbl.search("world").limit(5)
+```
+
+NOTE:
+
+You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
+
+## Multi-modal embedding function example
+You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support. LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
+
+```python
+@register("open-clip")
+class OpenClipEmbeddings(EmbeddingFunction):
+    name: str = "ViT-B-32"
+    pretrained: str = "laion2b_s34b_b79k"
+    device: str = "cpu"
+    batch_size: int = 64
+    normalize: bool = True
+    _model = PrivateAttr()
+    _preprocess = PrivateAttr()
+    _tokenizer = PrivateAttr()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        model, _, preprocess = open_clip.create_model_and_transforms(
+            self.name, pretrained=self.pretrained
+        )
+        model.to(self.device)
+        self._model, self._preprocess = model, preprocess
+        self._tokenizer = open_clip.get_tokenizer(self.name)
+        self._ndims = None
+
+    def ndims(self):
+        if self._ndims is None:
+            self._ndims = self.generate_text_embeddings("foo").shape[0]
+        return self._ndims
+
+    def compute_query_embeddings(
+        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            return [self.generate_text_embeddings(query)]
+        else:
+            PIL = attempt_import_or_raise("PIL", "pillow")
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError("OpenClip supports str or PIL Image as query")
+
+    def generate_text_embeddings(self, text: str) -> np.ndarray:
+        torch = attempt_import_or_raise("torch")
+        text = self.sanitize_input(text)
+        text = self._tokenizer(text)
+        text.to(self.device)
+        with torch.no_grad():
+            text_features = self._model.encode_text(text.to(self.device))
+            if self.normalize:
+                text_features /= text_features.norm(dim=-1, keepdim=True)
+            return text_features.cpu().numpy().squeeze()
+
+    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(images, (str, bytes)):
+            images = [images]
+        elif isinstance(images, pa.Array):
+            images = images.to_pylist()
+        elif isinstance(images, pa.ChunkedArray):
+            images = images.combine_chunks().to_pylist()
+        return images
+
+    def compute_source_embeddings(
+        self, images: IMAGES, *args, **kwargs
+    ) -> List[np.array]:
+        """
+        Get the embeddings for the given images
+        """
+        images = self.sanitize_input(images)
+        embeddings = []
+        for i in range(0, len(images), self.batch_size):
+            j = min(i + self.batch_size, len(images))
+            batch = images[i:j]
+            embeddings.extend(self._parallel_get(batch))
+        return embeddings
+
+    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+        """
+        Issue concurrent requests to retrieve the image data
+        """
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [
+                executor.submit(self.generate_image_embedding, image)
+                for image in images
+            ]
+            return [future.result() for future in futures]
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        torch = attempt_import_or_raise("torch")
+        # TODO handle retry and errors for https
+        image = self._to_pil(image)
+        image = self._preprocess(image).unsqueeze(0)
+        with torch.no_grad():
+            return self._encode_and_normalize_image(image)
+
+    def _to_pil(self, image: Union[str, bytes]):
+        PIL = attempt_import_or_raise("PIL", "pillow")
+        if isinstance(image, bytes):
+            return PIL.Image.open(io.BytesIO(image))
+        if isinstance(image, PIL.Image.Image):
+            return image
+        elif isinstance(image, str):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            if parsed.scheme == "file":
+                return PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                return PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+
+    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+        """
+        encode a single image tensor and optionally normalize the output
+        """
+        image_features = self._model.encode_image(image_tensor)
+        if self.normalize:
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+        return image_features.cpu().numpy().squeeze()
+```

docs/src/embeddings/default_embedding_functions.md

@@ -0,0 +1,632 @@
+There are various embedding functions available out of the box with LanceDB to manage your embeddings implicitly. We're actively working on adding other popular embedding APIs and models.
+
+## Text embedding functions
+Contains the text embedding functions registered by default.
+
+* Embedding functions have an inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential backoff. 
+* Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the default value of 7.
+
+### Sentence transformers
+Allows you to set parameters when registering a `sentence-transformers` object.
+
+!!! info
+    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
+| `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
+| `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
+
+
+??? "Check out available sentence-transformer models here!"
+    ```markdown
+    - sentence-transformers/all-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-mpnet-base-v2
+    - sentence-transformers/gtr-t5-base
+    - sentence-transformers/LaBSE
+    - sentence-transformers/all-MiniLM-L6-v2
+    - sentence-transformers/bert-base-nli-max-tokens
+    - sentence-transformers/bert-base-nli-mean-tokens
+    - sentence-transformers/bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
+    - sentence-transformers/bert-large-nli-cls-token
+    - sentence-transformers/bert-large-nli-max-tokens
+    - sentence-transformers/bert-large-nli-mean-tokens
+    - sentence-transformers/bert-large-nli-stsb-mean-tokens
+    - sentence-transformers/distilbert-base-nli-max-tokens
+    - sentence-transformers/distilbert-base-nli-mean-tokens
+    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
+    - sentence-transformers/distilroberta-base-msmarco-v1
+    - sentence-transformers/distilroberta-base-msmarco-v2
+    - sentence-transformers/nli-bert-base-cls-pooling
+    - sentence-transformers/nli-bert-base-max-pooling
+    - sentence-transformers/nli-bert-base
+    - sentence-transformers/nli-bert-large-cls-pooling
+    - sentence-transformers/nli-bert-large-max-pooling
+    - sentence-transformers/nli-bert-large
+    - sentence-transformers/nli-distilbert-base-max-pooling
+    - sentence-transformers/nli-distilbert-base
+    - sentence-transformers/nli-roberta-base
+    - sentence-transformers/nli-roberta-large
+    - sentence-transformers/roberta-base-nli-mean-tokens
+    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
+    - sentence-transformers/roberta-large-nli-mean-tokens
+    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
+    - sentence-transformers/stsb-bert-base
+    - sentence-transformers/stsb-bert-large
+    - sentence-transformers/stsb-distilbert-base
+    - sentence-transformers/stsb-roberta-base
+    - sentence-transformers/stsb-roberta-large
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
+    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
+    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
+    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
+    - sentence-transformers/bert-base-nli-cls-token
+    - sentence-transformers/all-distilroberta-v1
+    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
+    - sentence-transformers/multi-qa-distilbert-cos-v1
+    - sentence-transformers/multi-qa-distilbert-dot-v1
+    - sentence-transformers/multi-qa-mpnet-base-cos-v1
+    - sentence-transformers/multi-qa-mpnet-base-dot-v1
+    - sentence-transformers/nli-distilroberta-base-v2
+    - sentence-transformers/all-MiniLM-L6-v1
+    - sentence-transformers/all-mpnet-base-v1
+    - sentence-transformers/all-mpnet-base-v2
+    - sentence-transformers/all-roberta-large-v1
+    - sentence-transformers/allenai-specter
+    - sentence-transformers/average_word_embeddings_glove.6B.300d
+    - sentence-transformers/average_word_embeddings_glove.840B.300d
+    - sentence-transformers/average_word_embeddings_komninos
+    - sentence-transformers/average_word_embeddings_levy_dependency
+    - sentence-transformers/clip-ViT-B-32-multilingual-v1
+    - sentence-transformers/clip-ViT-B-32
+    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
+    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
+    - sentence-transformers/distilroberta-base-paraphrase-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v1
+    - sentence-transformers/distiluse-base-multilingual-cased-v2
+    - sentence-transformers/distiluse-base-multilingual-cased
+    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
+    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
+    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
+    - sentence-transformers/gtr-t5-large
+    - sentence-transformers/gtr-t5-xl
+    - sentence-transformers/gtr-t5-xxl
+    - sentence-transformers/msmarco-MiniLM-L-12-v3
+    - sentence-transformers/msmarco-MiniLM-L-6-v3
+    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
+    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
+    - sentence-transformers/msmarco-bert-base-dot-v5
+    - sentence-transformers/msmarco-bert-co-condensor
+    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
+    - sentence-transformers/msmarco-distilbert-base-tas-b
+    - sentence-transformers/msmarco-distilbert-base-v2
+    - sentence-transformers/msmarco-distilbert-base-v3
+    - sentence-transformers/msmarco-distilbert-base-v4
+    - sentence-transformers/msmarco-distilbert-cos-v5
+    - sentence-transformers/msmarco-distilbert-dot-v5
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
+    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
+    - sentence-transformers/msmarco-distilroberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-ance-firstp
+    - sentence-transformers/msmarco-roberta-base-v2
+    - sentence-transformers/msmarco-roberta-base-v3
+    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
+    - sentence-transformers/nli-mpnet-base-v2
+    - sentence-transformers/nli-roberta-base-v2
+    - sentence-transformers/nq-distilbert-base-v1
+    - sentence-transformers/paraphrase-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-MiniLM-L3-v2
+    - sentence-transformers/paraphrase-MiniLM-L6-v2
+    - sentence-transformers/paraphrase-TinyBERT-L6-v2
+    - sentence-transformers/paraphrase-albert-base-v2
+    - sentence-transformers/paraphrase-albert-small-v2
+    - sentence-transformers/paraphrase-distilroberta-base-v1
+    - sentence-transformers/paraphrase-distilroberta-base-v2
+    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
+    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
+    - sentence-transformers/quora-distilbert-base
+    - sentence-transformers/quora-distilbert-multilingual
+    - sentence-transformers/sentence-t5-base
+    - sentence-transformers/sentence-t5-large
+    - sentence-transformers/sentence-t5-xxl
+    - sentence-transformers/sentence-t5-xl
+    - sentence-transformers/stsb-distilroberta-base-v2
+    - sentence-transformers/stsb-mpnet-base-v2
+    - sentence-transformers/stsb-roberta-base-v2
+    - sentence-transformers/stsb-xlm-r-multilingual
+    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
+    - sentence-transformers/clip-ViT-L-14
+    - sentence-transformers/clip-ViT-B-16
+    - sentence-transformers/use-cmlm-multilingual
+    - sentence-transformers/all-MiniLM-L12-v1
+    ```
+
+!!! info
+    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
+    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
+
+!!! note "BAAI Embeddings example"
+    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+    class Words(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("words", schema=Words)
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+    )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
+
+
+### Huggingface embedding models
+We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
+
+Example usage - 
+```python
+import lancedb
+import pandas as pd
+
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+model = get_registry().get("huggingface").create(name='facebook/bart-base')
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
+table = db.create_table("greets", schema=Words)
+table.add()
+query = "old greeting"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+
+### Ollama embeddings
+Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
+
+- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
+- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
+
+| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
+|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
+| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
+| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
+| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("ollama").create(name="nomic-embed-text")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+
+### OpenAI embeddings
+LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
+| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### Instructor Embeddings
+[Instructor](https://instructor-embedding.github.io/) is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g. classification, retrieval, clustering, text evaluation, etc.) and domains (e.g. science, finance, etc.) by simply providing the task instruction, without any finetuning.
+
+If you want to calculate customized embeddings for specific sentences, you can follow the unified template to write instructions.
+
+!!! info
+    Represent the `domain` `text_type` for `task_objective`:
+
+    * `domain` is optional, and it specifies the domain of the text, e.g. science, finance, medicine, etc.
+    * `text_type` is required, and it specifies the encoding unit, e.g. sentence, document, paragraph, etc.
+    * `task_objective` is optional, and it specifies the objective of embedding, e.g. retrieve a document, classify the sentence, etc.
+
+More information about the model can be found at the [source URL](https://github.com/xlang-ai/instructor-embedding).
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
+
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
+### Gemini Embeddings
+With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
+The Gemini Embedding Model API supports various task types:
+
+| Task Type               | Description                                                                                                                                                |
+|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| "`retrieval_query`"     | Specifies the given text is a query in a search/retrieval setting.                                                                                         |
+| "`retrieval_document`"  | Specifies the given text is a document in a search/retrieval setting. Using this task type requires a title but is automatically proided by Embeddings API |
+| "`semantic_similarity`" | Specifies the given text will be used for Semantic Textual Similarity (STS).                                                                               |
+| "`classification`"      | Specifies that the embeddings will be used for classification.                                                                                             |
+| "`clusering`"           | Specifies that the embeddings will be used for clustering.                                                                                                 |
+
+
+Usage Example:
+
+```python
+import lancedb
+import pandas as pd
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+
+model = get_registry().get("gemini-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
+### Cohere Embeddings
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+```
+    * embed-english-v3.0
+    * embed-multilingual-v3.0
+    * embed-english-light-v3.0
+    * embed-multilingual-light-v3.0
+    * embed-english-v2.0
+    * embed-english-light-v2.0
+    * embed-multilingual-v2.0
+```
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+    
+    ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import EmbeddingFunctionRegistry
+
+        cohere = EmbeddingFunctionRegistry
+            .get_instance()
+            .get("cohere")
+            .create(name="embed-multilingual-v2.0")
+
+        class TextModel(LanceModel):
+            text: str = cohere.SourceField()
+            vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+        data = [ { "text": "hello world" },
+                { "text": "goodbye world" }]
+
+        db = lancedb.connect("~/.lancedb")
+        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+        tbl.add(data)
+    ```
+
+### AWS Bedrock Text Embedding Functions
+AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
+You can do so by using `awscli` and also add your session_token:
+```shell
+aws configure
+aws configure set aws_session_token "<your_session_token>"
+```
+to ensure that the credentials are set up correctly, you can run the following command:
+```shell
+aws sts get-caller-identity
+```
+
+Supported Embedding modelIDs are:
+* `amazon.titan-embed-text-v1`
+* `cohere.embed-english-v3`
+* `cohere.embed-multilingual-v3`
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
+| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
+| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
+| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
+| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
+| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
+| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
+
+Usage Example:
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+model = get_registry().get("bedrock-text").create()
+
+class TextModel(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+db = lancedb.connect("tmp_path")
+tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+tbl.add(df)
+rs = tbl.search("hello").limit(1).to_pandas()
+```
+
+## Multi-modal embedding functions
+Multi-modal embedding functions allow you to query your table using both images and text.
+
+### OpenClip embeddings
+We support CLIP model embeddings using the open source alternative, [open-clip](https://github.com/mlfoundations/open_clip) which supports various customizations. It is registered as `open-clip` and supports the following customizations:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"ViT-B-32"` | The name of the model. |
+| `pretrained` | `str` | `"laion2b_s34b_b79k"` | The name of the pretrained model to load. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `batch_size` | `int` | `64` | The number of images to process in a batch. |
+| `normalize` | `bool` | `True` | Whether to normalize the input images before feeding them to the model. |
+
+This embedding function supports ingesting images as both bytes and urls. You can query them using both test and other images.
+
+!!! info
+    LanceDB supports ingesting images directly from accessible links.
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect(tmp_path)
+func = get_registry.get("open-clip").create()
+
+class Images(LanceModel):
+    label: str
+    image_uri: str = func.SourceField() # image uri as the source
+    image_bytes: bytes = func.SourceField() # image bytes as the source
+    vector: Vector(func.ndims()) = func.VectorField() # vector column 
+    vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column 
+
+table = db.create_table("images", schema=Images)
+labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+uris = [
+    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+]
+# get each uri as bytes
+image_bytes = [requests.get(uri).content for uri in uris]
+table.add(
+    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+)
+```
+Now we can search using text from both the default vector column and the custom vector column
+```python
+
+# text search
+actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
+print(actual.label) # prints "dog"
+
+frombytes = (
+    table.search("man's best friend", vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(frombytes.label)
+
+```
+
+Because we're using a multi-modal embedding function, we can also search using images
+
+```python
+# image search
+query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
+image_bytes = requests.get(query_image_uri).content
+query_image = Image.open(io.BytesIO(image_bytes))
+actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
+print(actual.label == "dog")
+
+# image search using a custom vector column
+other = (
+    table.search(query_image, vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(actual.label)
+
+```
+
+### Imagebind embeddings
+We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
+
+This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"imagebind_huge"` | Name of the model. |
+| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
+| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
+
+Below is an example demonstrating how the API works:
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect(tmp_path)
+func = get_registry.get("imagebind").create()
+
+class ImageBindModel(LanceModel):
+    text: str
+    image_uri: str = func.SourceField()
+    audio_path: str
+    vector: Vector(func.ndims()) = func.VectorField()
+
+# add locally accessible image paths
+text_list=["A dog.", "A car", "A bird"]
+image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
+audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
+
+# Load data
+inputs = [
+    {"text": a, "audio_path": b, "image_uri": c}
+    for a, b, c in zip(text_list, audio_paths, image_paths)
+]
+
+#create table and add data
+table = db.create_table("img_bind", schema=ImageBindModel)
+table.add(inputs)
+```
+
+Now, we can search using any modality:
+
+#### image search
+```python
+query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
+actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "dog")
+```
+#### audio search
+
+```python
+query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
+actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "car")
+```
+#### Text search
+You can add any input query and fetch the result as follows:
+```python
+query = "an animal which flies and tweets" 
+actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
+print(actual.text == "bird")
+```
+
+If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_functions.md

@@ -0,0 +1,172 @@
+Representing multi-modal data as vector embeddings is becoming a standard practice. Embedding functions can themselves be thought of as key part of the data processing pipeline that each request has to be passed through. The assumption here is: after initial setup, these components and the underlying methodology are not expected to change for a particular project.
+
+For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
+
+!!! Note "LanceDB cloud doesn't support embedding functions yet"
+    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
+
+!!! warning
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
+    table metadata and have LanceDB automatically take care of regenerating the embeddings.
+
+
+## 1. Define the embedding function
+
+=== "Python"
+    In the LanceDB python SDK, we define a global embedding function registry with
+    many different embedding models and even more coming soon. 
+    Here's let's an implementation of CLIP as example.
+
+    ```python
+    from lancedb.embeddings import get_registry
+
+    registry = get_registry()
+    clip = registry.get("open-clip").create()
+    ```
+
+    You can also define your own embedding function by implementing the `EmbeddingFunction` 
+    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
+
+=== "JavaScript""
+    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    // You need to provide an OpenAI API key
+    const apiKey = "sk-..."
+    // The embedding function will create embeddings for the 'text' column
+    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    ```
+
+## 2. Define the data model or schema
+
+=== "Python"
+    The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
+
+    ```python
+    class Pets(LanceModel):
+        vector: Vector(clip.ndims()) = clip.VectorField()
+        image_uri: str = clip.SourceField()
+    ```
+
+    `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
+
+=== "JavaScript"
+
+    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
+    Arrow schema can be provided.
+
+## 3. Create table and add data
+
+Now that we have chosen/defined our embedding function and the schema, 
+we can create the table and ingest data without needing to explicitly generate
+the embeddings at all:
+
+=== "Python"
+    ```python
+    db = lancedb.connect("~/lancedb")
+    table = db.create_table("pets", schema=Pets)
+
+    table.add([{"image_uri": u} for u in uris])
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
+
+    const table = await db.createTable("vectors", data, embedding)
+    ```
+
+## 4. Querying your table
+Not only can you forget about the embeddings during ingestion, you also don't
+need to worry about it when you query the table:
+
+=== "Python"
+
+    Our OpenCLIP query embedding function supports querying via both text and images:
+
+    ```python
+    results = (
+        table.search("dog")
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Or we can search using an image:
+
+    ```python
+    p = Path("path/to/images/samoyed_100.jpg")
+    query_image = Image.open(p)
+    results = (
+        table.search(query_image)
+        .limit(10)
+        .to_pandas()
+    )
+    ```
+
+    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "JavaScript"
+
+    ```javascript
+    const results = await table
+    .search("What's the best pizza topping?")
+    .limit(10)
+    .execute()
+    ```    
+    
+    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
+
+---
+
+## Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default, the maximum retires is set to 7. You can tune it by setting it to a different number, or disable it by setting it to 0.
+
+An example of how to do this is shown below:
+
+```python
+clip = registry.get("open-clip").create() # Defaults to 7 max retries
+clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
+clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
+```
+
+!!! note
+    Embedding functions can also fail due to other errors that have nothing to do with rate limits.
+    This is why the error is also logged.
+
+## Some fun with Pydantic
+
+LanceDB is integrated with Pydantic, which was used in the example above to define the schema in Python. It's also used behind the scenes by the embedding function API to ingest useful information as table metadata.
+
+You can also use the integration for adding utility operations in the schema. For example, in our multi-modal example, you can search images using text or another image. Let's define a utility function to plot the image.
+
+```python
+class Pets(LanceModel):
+    vector: Vector(clip.ndims()) = clip.VectorField()
+    image_uri: str = clip.SourceField()
+
+    @property
+    def image(self):
+        return Image.open(self.image_uri)
+```
+Now, you can covert your search results to a Pydantic model and use this property.
+
+```python
+rs = table.search(query_image).limit(3).to_pydantic(Pets)
+rs[2].image
+```
+
+![](../assets/dog_clip_output.png)
+
+Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
+let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -0,0 +1,74 @@
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+This makes them a very powerful tool for machine learning practitioners. 
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
+(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
+
+LanceDB supports 3 methods of working with embeddings.
+
+1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
+2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
+3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+   that extends the default embedding functions.
+
+For python users, there is also a legacy [with_embeddings API](./legacy.md).
+It is retained for compatibility and will be removed in a future version.
+
+## Quickstart
+
+To get started with embeddings, you can use the built-in embedding functions.
+
+### OpenAI Embedding function
+LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+### Sentence Transformers Embedding function
+LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+class Words(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+table = db.create_table("words", schema=Words)
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+)
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```

docs/src/embeddings/legacy.md

@@ -0,0 +1,99 @@
+The legacy `with_embeddings` API is for Python only and is deprecated.
+
+### Hugging Face
+
+The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
+library, which can be installed via pip.
+
+```bash
+pip install sentence-transformers
+```
+
+The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
+for a given document.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+name="paraphrase-albert-small-v2"
+model = SentenceTransformer(name)
+
+# used for both training and querying
+def embed_func(batch):
+    return [model.encode(sentence) for sentence in batch]
+```
+
+
+### OpenAI
+
+Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
+
+```python
+import openai
+import os
+
+# Configuring the environment variable OPENAI_API_KEY
+if "OPENAI_API_KEY" not in os.environ:
+# OR set the key here as a variable
+openai.api_key = "sk-..."
+
+client = openai.OpenAI()
+
+def embed_func(c):    
+    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
+    return [record.embedding for record in rs["data"]]
+```
+
+
+## Applying an embedding function to data
+
+Using an embedding function, you can apply it to raw data
+to generate embeddings for each record.
+
+Say you have a pandas DataFrame with a `text` column that you want embedded,
+you can use the `with_embeddings` function to generate embeddings and add them to 
+an existing table.
+
+```python
+    import pandas as pd
+    from lancedb.embeddings import with_embeddings
+
+    df = pd.DataFrame(
+        [
+            {"text": "pepperoni"},
+            {"text": "pineapple"}
+        ]
+    )
+    data = with_embeddings(embed_func, df)
+
+    # The output is used to create / append to a table
+    tbl = db.create_table("my_table", data=data)
+```
+
+If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+
+By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+using the `batch_size` parameter to `with_embeddings`.
+
+LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+API call is reliable.
+
+## Querying using an embedding function
+
+!!! warning
+    At query time, you **must** use the same embedding function you used to vectorize your data.
+    If you use a different embedding function, the embeddings will not reside in the same vector
+    space and the results will be nonsensical.
+
+=== "Python"
+     ```python
+     query = "What's the best pizza topping?"
+     query_vector = embed_func([query])[0]
+     results = (
+        tbl.search(query_vector)
+        .limit(10)
+        .to_pandas()
+     )
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/examples_js.md

@@ -1,7 +1,6 @@
-# Examples
+# Examples: JavaScript
 
-Here are some of the examples, projects and applications using vectordb native javascript library.
-Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+To help you get started, we provide some examples, projects and applications that use the LanceDB JavaScript API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
 
 | Example | Scripts  |
 |-------- | ------   |
@@ -10,10 +9,3 @@ Some examples are covered in detail in the next sections. You can find more on [
 | [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
 | [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
 | [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js)  |
-
-## Projects & Applications
-
-| Project Name                                        | Description                                                                                                          | Screenshot                                |
-|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
-| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
-| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |

docs/src/examples/examples_python.md

@@ -0,0 +1,17 @@
+# Examples: Python
+
+To help you get started, we provide some examples, projects and applications that use the LanceDB Python API. You can always find the latest examples in our [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes) repository.
+
+| Example | Interactive Envs | Scripts  |
+|-------- | ---------------- | ------   |
+| | | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
+| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
+| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
+| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
+| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
+| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
+| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
+

docs/src/examples/examples_rust.md

@@ -0,0 +1,3 @@
+# Examples: Rust
+
+Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -0,0 +1,165 @@
+# How to Load Image Embeddings into LanceDB
+
+With the rise of Large Multimodal Models (LMMs) such as [GPT-4 Vision](https://blog.roboflow.com/gpt-4-vision/), the need for storing image embeddings is growing. The most effective way to store text and image embeddings is in a vector database such as LanceDB. Vector databases are a special kind of data store that enables efficient search over stored embeddings. 
+
+[CLIP](https://blog.roboflow.com/openai-clip/), a multimodal model developed by OpenAI, is commonly used to calculate image embeddings. These embeddings can then be used with a vector database to build a semantic search engine that you can query using images or text. For example, you could use LanceDB and CLIP embeddings to build a search engine for a database of folders.
+
+In this guide, we are going to show you how to use Roboflow Inference to load image embeddings into LanceDB. Without further ado, let’s get started!
+
+## Step #1: Install Roboflow Inference
+
+[Roboflow Inference](https://inference.roboflow.com) enables you to run state-of-the-art computer vision models with minimal configuration. Inference supports a range of models, from fine-tuned object detection, classification, and segmentation models to foundation models like CLIP. We will use Inference to calculate CLIP image embeddings.
+
+Inference provides a HTTP API through which you can run vision models.
+
+Inference powers the Roboflow hosted API, and is available as an open source utility. In this guide, we are going to run Inference locally, which enables you to calculate CLIP embeddings on your own hardware. We will also show you how to use the hosted Roboflow CLIP API, which is ideal if you need to scale and do not want to manage a system for calculating embeddings.
+
+To get started, first install the Inference CLI:
+
+```
+pip install inference-cli
+```
+
+Next, install Docker. Refer to the official Docker installation instructions for your operating system to get Docker set up. Once Docker is ready, you can start Inference using the following command:
+
+```
+inference server start
+```
+
+An Inference server will start running at ‘http://localhost:9001’.
+
+## Step #2: Set Up a LanceDB Vector Database
+
+Now that we have Inference running, we can set up a LanceDB vector database. You can run LanceDB in JavaScript and Python. For this guide, we will use the Python API. But, you can take the HTTP requests we make below and change them to JavaScript if required.
+
+For this guide, we are going to search the [COCO 128 dataset](https://universe.roboflow.com/team-roboflow/coco-128), which contains a wide range of objects. The variability in objects present in this dataset makes it a good dataset to demonstrate the capabilities of vector search. If you want to use this dataset, you can download [COCO 128 from Roboflow Universe](https://universe.roboflow.com/team-roboflow/coco-128). With that said, you can search whatever folder of images you want.
+
+Once you have a dataset ready, install LanceDB with the following command:
+
+```
+pip install lancedb
+```
+
+We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
+
+```
+pip install tantivy
+```
+
+Create a new Python file and add the following code:
+
+```python
+import cv2
+import supervision as sv
+import requests
+
+import lancedb
+
+db = lancedb.connect("./embeddings")
+
+IMAGE_DIR = "images/"
+API_KEY = os.environ.get("ROBOFLOW_API_KEY")
+SERVER_URL = "http://localhost:9001"
+
+results = []
+
+for i, image in enumerate(os.listdir(IMAGE_DIR)):
+    infer_clip_payload = {
+        #Images can be provided as urls or as base64 encoded strings
+        "image": {
+            "type": "base64",
+            "value": base64.b64encode(open(IMAGE_DIR + image, "rb").read()).decode("utf-8"),
+        },
+    }
+
+    res = requests.post(
+        f"{SERVER_URL}/clip/embed_image?api_key={API_KEY}",
+        json=infer_clip_payload,
+    )
+
+    embeddings = res.json()['embeddings']
+
+    print("Calculated embedding for image: ", image)
+
+    image = {"vector": embeddings[0], "name": os.path.join(IMAGE_DIR, image)}
+
+    results.append(image)
+
+tbl = db.create_table("images", data=results)
+
+tbl.create_fts_index("name")
+```
+
+To use the code above, you will need a Roboflow API key. [Learn how to retrieve a Roboflow API key](https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key). Run the following command to set up your API key in your environment:
+
+```
+export ROBOFLOW_API_KEY=""
+```
+
+Replace the `IMAGE_DIR` value with the folder in which you are storing the images for which you want to calculate embeddings. If you want to use the Roboflow CLIP API to calculate embeddings, replace the `SERVER_URL` value with `https://infer.roboflow.com`.
+
+Run the script above to create a new LanceDB database. This database will be stored on your local machine. The database will be called `embeddings` and the table will be called `images`.
+
+The script above calculates all embeddings for a folder then creates a new table. To add additional images, use the following code:
+
+```python
+def make_batches():
+    for i in range(5):
+        yield [
+                {"vector": [3.1, 4.1], "name": "image1.png"},
+                {"vector": [5.9, 26.5], "name": "image2.png"}
+            ]
+
+tbl = db.open_table("images")
+tbl.add(make_batches())
+```
+
+Replacing the `make_batches()` function with code to load embeddings for images.
+
+## Step #3: Run a Search Query
+
+We are now ready to run a search query. To run a search query, we need a text embedding that represents a text query. We can use this embedding to search our LanceDB database for an entry.
+
+Let’s calculate a text embedding for the query “cat”, then run a search query:
+
+```python
+infer_clip_payload = {
+    "text": "cat",
+}
+
+res = requests.post(
+    f"{SERVER_URL}/clip/embed_text?api_key={API_KEY}",
+    json=infer_clip_payload,
+)
+
+embeddings = res.json()['embeddings']
+
+df = tbl.search(embeddings[0]).limit(3).to_list()
+
+print("Results:")
+
+for i in df:
+    print(i["name"])
+```
+
+This code will search for the three images most closely related to the prompt “cat”. The names of the most similar three images will be printed to the console. Here are the three top results:
+
+```
+dataset/images/train/000000000650_jpg.rf.1b74ba165c5a3513a3211d4a80b69e1c.jpg
+dataset/images/train/000000000138_jpg.rf.af439ef1c55dd8a4e4b142d186b9c957.jpg
+dataset/images/train/000000000165_jpg.rf.eae14d5509bf0c9ceccddbb53a5f0c66.jpg
+```
+
+Let’s open the top image:
+
+![Cat](https://media.roboflow.com/cat_lancedb.jpg)
+
+The top image was a cat. Our search was successful.
+
+## Conclusion
+
+LanceDB is a vector database that you can use to store and efficiently search your image embeddings. You can use Roboflow Inference, a scalable computer vision inference server, to calculate CLIP embeddings that you can store in LanceDB.
+
+You can use Inference and LanceDB together to build a range of applications with image embeddings, from a media search engine to a retrieval-augmented generation pipeline for use with LMMs.
+
+To learn more about Inference and its capabilities, refer to the Inference documentation.

docs/src/examples/index.md

@@ -1,21 +1,14 @@
-# Examples
+# Example projects and recipes
 
-Here are some of the examples, projects and applications using LanceDB python library. Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+## Recipes and example code
 
-| Example | Interactive Envs | Scripts  |
-|-------- | ---------------- | ------   |
-| | | |
-| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
-| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
-| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/reducing_hallucinations_ai_agents/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
-| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py)  |
-| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py)  |
-| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/movie-recommender/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py)  |
-| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/audio_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py)  |
-| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> | [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py)  |
-| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | <a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/prompttools-eval-prompts/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a> |  |
+LanceDB provides language APIs, allowing you to embed a database in your language of choice.
 
-## Projects & Applications powered by LanceDB
+* 🐍 [Python](examples_python.md) examples
+* 👾 [JavaScript](examples_js.md) examples
+* 🦀 Rust examples (coming soon)
+
+## Applications powered by LanceDB
 
 | Project Name                                        | Description                                                                                                          | Screenshot                                |
 |-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|

docs/src/examples/modal_langchain.py

@@ -1,6 +1,5 @@
 import pickle
 import re
-import sys
 import zipfile
 from pathlib import Path
 
@@ -79,7 +78,10 @@ def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000,
+        chunk_overlap=200,
+    )
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 

docs/src/examples/serverless_lancedb_with_s3_and_lambda.md

@@ -80,14 +80,14 @@ def handler(event, context):
     # Shape of SIFT is (128,1M), d=float32
     query_vector = np.array(event['query_vector'], dtype=np.float32)
 
-    rs = table.search(query_vector).limit(2).to_df()
+    rs = table.search(query_vector).limit(2).to_list()
 
     return {
         "statusCode": status_code,
         "headers": {
             "Content-Type": "application/json"
         },
-        "body": rs.to_json()
+        "body": json.dumps(rs)
     }
 ``` 
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -0,0 +1,79 @@
+// Creates an SVG robot icon (from Lucide)
+function robotSVG() {
+  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+  svg.setAttribute("width", "24");
+  svg.setAttribute("height", "24");
+  svg.setAttribute("viewBox", "0 0 24 24");
+  svg.setAttribute("fill", "none");
+  svg.setAttribute("stroke", "currentColor");
+  svg.setAttribute("stroke-width", "2");
+  svg.setAttribute("stroke-linecap", "round");
+  svg.setAttribute("stroke-linejoin", "round");
+  svg.setAttribute("class", "lucide lucide-bot-message-square");
+
+  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path1.setAttribute("d", "M12 6V2H8");
+  svg.appendChild(path1);
+
+  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
+  svg.appendChild(path2);
+
+  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path3.setAttribute("d", "M2 12h2");
+  svg.appendChild(path3);
+
+  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path4.setAttribute("d", "M9 11v2");
+  svg.appendChild(path4);
+
+  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path5.setAttribute("d", "M15 11v2");
+  svg.appendChild(path5);
+
+  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
+  path6.setAttribute("d", "M20 12h2");
+  svg.appendChild(path6);
+
+  return svg
+}
+
+// Creates the Fluidic Chatbot buttom
+function fluidicButton() {
+  var btn = document.createElement("a");
+  btn.href = "https://asklancedb.com";
+  btn.target = "_blank";
+  btn.style.position = "fixed";
+  btn.style.fontWeight = "bold";
+  btn.style.fontSize = ".8rem";
+  btn.style.right = "10px";
+  btn.style.bottom = "10px";
+  btn.style.width = "80px";
+  btn.style.height = "80px";
+  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
+  btn.style.color = "white";
+  btn.style.borderRadius = "5px";
+  btn.style.display = "flex";
+  btn.style.flexDirection = "column";
+  btn.style.justifyContent = "center";
+  btn.style.alignItems = "center";
+  btn.style.zIndex = "1000";
+  btn.style.opacity = "0";
+  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
+  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
+
+  setTimeout(function() {
+      btn.style.opacity = "1";
+      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
+  }, 0);
+
+  return btn
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+  var btn = fluidicButton()
+  btn.appendChild(robotSVG());
+  var text = document.createTextNode("Ask AI");
+  btn.appendChild(text);
+  document.body.appendChild(btn);
+});

docs/src/faq.md

@@ -0,0 +1,87 @@
+This section covers some common questions and issues that you may encounter when using LanceDB.
+
+### Is LanceDB open source?
+
+Yes, LanceDB is an open source vector database available under an Apache 2.0 license. We also have a serverless SaaS solution, LanceDB Cloud, available under a commercial license.
+
+### What is the difference between Lance and LanceDB?
+
+[Lance](https://github.com/lancedb/lance) is a modern columnar data format for AI, written in Rust 🦀. It’s perfect for building search engines, feature stores and being the foundation of large-scale ML training jobs requiring high performance IO and shuffles. It also has native support for storing, querying, and inspecting deeply nested data for robotics or large blobs like images, point clouds, and more.
+
+LanceDB is the vector database that’s built on top of Lance, and utilizes the underlying optimized storage format to build efficient disk-based indexes that power semantic search & retrieval applications, from RAGs to QA Bots to recommender systems.
+
+### Why invent another data format instead of using Parquet?
+
+As we mention in our talk titled “[Lance, a modern columnar data format](https://www.youtube.com/watch?v=ixpbVyrsuL8)”, Parquet and other tabular formats that derive from it are rather dated (Parquet is over 10 years old), especially when it comes to random access on vectors. We needed a format that’s able to handle the complex trade-offs involved in shuffling, scanning, OLAP and filtering large datasets involving vectors, and our extensive experiments with Parquet didn't yield sufficient levels of performance for modern ML. [Our benchmarks](https://blog.lancedb.com/benchmarking-random-access-in-lance-ed690757a826) show that Lance is up to 1000x faster than Parquet for random access, which we believe justifies our decision to create a new data format for AI.
+
+### Why build in Rust? 🦀
+
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
+
+### What is the difference between LanceDB OSS and LanceDB Cloud?
+
+LanceDB OSS is an **embedded** (in-process) solution that can be used as the vector store of choice for your LLM and RAG applications. It can be embedded inside an existing application backend, or used in-process alongside existing ML and data engineering pipelines.
+
+LanceDB Cloud is a **serverless** solution — the database and data sit on the cloud and we manage the scalability of the application side via a remote client, without the need to manage any infrastructure.
+
+Both flavors of LanceDB benefit from the blazing fast Lance data format and are built on the same open source foundations.
+
+### What makes LanceDB different?
+
+LanceDB is among the few embedded vector DBs out there that we believe can unlock a whole new class of LLM-powered applications in the browser or via edge functions. Lance’s multi-modal nature allows you to store the raw data, metadata and the embeddings all at once, unlike other solutions that typically store just the embeddings and metadata.
+
+The Lance data format that powers our storage system also provides true zero-copy access and seamless interoperability with numerous other data formats (like Pandas, Polars, Pydantic) via Apache Arrow, as well as automatic data versioning and data management without needing extra infrastructure.
+
+### How large of a dataset can LanceDB handle?
+
+LanceDB and its underlying data format, Lance, are built to scale to really large amounts of data (hundreds of terabytes). We are currently working with customers who regularly perform operations on 200M+ vectors, and we’re fast approaching billion scale and beyond, which are well-handled by our disk-based indexes, without you having to break the bank.
+
+### Do I need to build an ANN index to run vector search?
+
+No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
+
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
+
+### Does LanceDB support full-text search?
+
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
+
+### How can I speed up data inserts?
+
+It's highly recommend to perform bulk inserts via batches (for e.g., Pandas DataFrames or lists of dicts in Python) to speed up inserts for large datasets. Inserting records one at a time is slow and can result in suboptimal performance because each insert creates a new data fragment on disk. Batching inserts allows LanceDB to create larger fragments (and their associated manifests), which are more efficient to read and write.
+
+### Do I need to set a refine factor when using an index?
+
+Yes. LanceDB uses PQ, or Product Quantization, to compress vectors and speed up search when using an ANN index. However, because PQ is a lossy compression algorithm, it tends to reduce recall while also reducing the index size. To address this trade-off, we introduce a process called **refinement**. The normal process computes distances by operating on the compressed PQ vectors. The refinement factor (*rf*) is a multiplier that takes the top-k similar PQ vectors to a given query, fetches `rf * k` *full* vectors and computes the raw vector distances between them and the query vector, reordering the top-k results based on these scores instead.
+
+For example, if you're retrieving the top 10 results and set `refine_factor` to 25, LanceDB will fetch the 250 most similar vectors (according to PQ), compute the distances again based on the full vectors for those 250 and then re-rank based on their scores. This can significantly improve recall, with a small added latency cost (typically a few milliseconds), so it's recommended you set a `refine_factor` of anywhere between 5-50 and measure its impact on latency prior to deploying your solution.
+
+### How can I improve IVF-PQ recall while keeping latency low?
+
+When using an IVF-PQ index, there's a trade-off between recall and latency at query time. You can improve recall by increasing the number of probes and the `refine_factor`. In our benchmark on the GIST-1M dataset, we show that it's possible to achieve >0.95 recall with a latency of under 10 ms on most systems, using ~50 probes and a `refine_factor` of 50. This is, of course, subject to the dataset at hand and a quick sensitivity study can be performed on your own data. You can find more details on the benchmark in our [blog post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a).
+
+![](assets/recall-vs-latency.webp)
+
+### How do I connect to MinIO?
+
+MinIO supports an S3 compatible API. In order to connect to a MinIO instance, you need to:
+
+- Set the envvar `AWS_ENDPOINT` to the URL of your MinIO API
+- Set the envvars `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your MinIO credential
+- Call `lancedb.connect("s3://minio_bucket_name")`
+
+### Where can I find benchmarks for LanceDB?
+
+Refer to this [post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a) for recent benchmarks.
+
+### How much data can LanceDB practically manage without effecting performance?
+
+We target good performance on ~10-50 billion rows and ~10-30 TB of data.
+
+### Does LanceDB support concurrent operations?
+
+LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the [storage layer](https://lancedb.github.io/lancedb/concepts/storage/) you've chosen scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit
+
+!!! info "Multiprocessing with LanceDB"
+
+    For multiprocessing you should probably not use ```fork``` as lance is multi-threaded internally and ```fork``` and multi-thread do not work well.[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)

docs/src/fts.md

@@ -1,26 +1,20 @@
-# [EXPERIMENTAL] Full text search
+# Full-text search
+
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
-LanceDB now provides experimental support for full text search.
-This is currently Python only. We plan to push the integration down to Rust in the future
-to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install the dependency `tantivy-py`:
+To use full-text search, install the dependency [`tantivy-py`](https://github.com/quickwit-oss/tantivy-py):
 
-# tantivy 0.20.1
 ```sh
+# Say you want to use tantivy==0.20.1
 pip install tantivy==0.20.1
 ```
 
+## Example
 
-## Quickstart
-
-Assume:
-1. `table` is a LanceDB Table
-2. `text` is the name of the `Table` column that we want to index
-
-For example,
+Consider that we have a LanceDB table named `my_table`, whose string column `text` we want to index and query via keyword search.
 
 ```python
 import lancedb
@@ -28,29 +22,51 @@ import lancedb
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-table = db.create_table("my_table",
-            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
-                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
-
+table = db.create_table(
+    "my_table",
+    data=[
+        {"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+        {"vector": [5.9, 26.5], "text": "There are several kittens playing"},
+    ],
+)
 ```
 
-To create the index:
+## Create FTS index on single column
+
+The FTS index must be created before you can search via keywords.
 
 ```python
 table.create_fts_index("text")
 ```
 
-To search:
+To search an FTS index via keywords, LanceDB's `table.search` accepts a string as input:
 
 ```python
-df = table.search("puppy").limit(10).select(["text"]).to_df()
+table.search("puppy").limit(10).select(["text"]).to_list()
 ```
 
-LanceDB automatically looks for an FTS index if the input is str.
+This returns the result as a list of dictionaries as follows.
 
-## Multiple text columns
+```python
+[{'text': 'Frodo was a happy puppy', 'score': 0.6931471824645996}]
+```
 
-If you have multiple columns to index, pass them all as a list to `create_fts_index`:
+!!! note
+    LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
+
+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("text", tokenizer_name="en_stem")
+```
+
+The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+
+## Index multiple columns
+
+If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
 
 ```python
 table.create_fts_index(["text1", "text2"])
@@ -58,10 +74,100 @@ table.create_fts_index(["text1", "text2"])
 
 Note that the search API call does not change - you can search over all indexed columns at once.
 
+## Filtering
+
+Currently the LanceDB full text search feature supports *post-filtering*, meaning filters are
+applied on top of the full text search results. This can be invoked via the familiar
+`where` syntax:
+
+```python
+table.search("puppy").limit(10).where("meta='foo'").to_list()
+```
+
+## Sorting
+
+You can pre-sort the documents by specifying `ordering_field_names` when
+creating the full-text search index. Once pre-sorted, you can then specify
+`ordering_field_name` while searching to return results sorted by the given
+field. For example, 
+
+```
+table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
+
+(table.search("terms", ordering_field_name="sort_by_field")
+ .limit(20)
+ .to_list())
+```
+
+!!! note
+    If you wish to specify an ordering field at query time, you must also
+    have specified it during indexing time. Otherwise at query time, an
+    error will be raised that looks like `ValueError: The field does not exist: xxx`
+
+!!! note
+    The fields to sort on must be of typed unsigned integer, or else you will see 
+    an error during indexing that looks like 
+    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
+
+!!! note
+    You can specify multiple fields for ordering at indexing time.
+    But at query time only one ordering field is supported.
+
+
+## Phrase queries vs. terms queries
+
+For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
+or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
+query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
+
+!!! tip "Note"
+    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
+
+    ```py
+    # This raises a syntax error
+    table.search("they could have been dogs OR cats")
+    ```
+
+    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
+    the query is treated as a phrase query.
+
+    ```py
+    # This works!
+    table.search("they could have been dogs or cats")
+    ```
+
+It can be cumbersome to have to remember what will cause a syntax error depending on the type of
+query you want to perform. To make this simpler, when you want to perform a phrase query, you can
+enforce it in one of two ways:
+
+1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
+a phrase query.
+2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
+itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
+is treated as a phrase query.
+
+In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
+double quotes replaced by single quotes.
+
+
+## Configurations
+
+By default, LanceDB configures a 1GB heap size limit for creating the index. You can
+reduce this if running on a smaller node, or increase this for faster performance while
+indexing a larger corpus.
+
+```python
+# configure a 512MB heap size
+heap = 1024 * 1024 * 512
+table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
+```
+
 ## Current limitations
 
 1. Currently we do not yet support incremental writes.
-If you add data after fts index creation, it won't be reflected
+   If you add data after FTS index creation, it won't be reflected
    in search results until you do a full reindex.
 
-2. We currently only support local filesystem paths for the fts index.
+2. We currently only support local filesystem paths for the FTS index.
+   This is a tantivy limitation. We've implemented an object store plugin
+   but there's no way in tantivy-py to specify to use it.

docs/src/guides/storage.md

@@ -0,0 +1,448 @@
+# Configuring cloud storage
+
+<!-- TODO: When we add documentation for how to configure other storage types
+           we can change the name to a more general "Configuring storage" -->
+
+When using LanceDB OSS, you can choose where to store your data. The tradeoffs between different storage options are discussed in the [storage concepts guide](../concepts/storage.md). This guide shows how to configure LanceDB to use different storage options.
+
+## Object Stores
+
+LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure Blob Store, and Google Cloud Storage. Which object store to use is determined by the URI scheme of the dataset path. `s3://` is used for AWS S3, `az://` is used for Azure Blob Storage, and `gs://` is used for Google Cloud Storage. These URIs are passed to the `connect` function:
+
+=== "Python"
+
+    AWS S3:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("s3://bucket/path")
+    ```
+
+    Google Cloud Storage:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("gs://bucket/path")
+    ```
+
+    Azure Blob Storage:
+
+    ```python
+    import lancedb
+    db = lancedb.connect("az://bucket/path")
+    ```
+
+=== "JavaScript"
+
+    AWS S3:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path");
+    ```
+
+    Google Cloud Storage:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("gs://bucket/path");
+    ```
+
+    Azure Blob Storage:
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("az://bucket/path");
+    ```
+
+In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
+
+```bash
+export TIMEOUT=60s
+```
+
+!!! note "`storage_options` availability"
+
+    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
+    It is not yet supported in the Python synchronous API.
+
+If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"}
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path",
+                                     {storageOptions: {timeout: "60s"}});
+    ```
+
+Getting even more specific, you can set the `timeout` for only a particular table:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async("s3://bucket/path")
+    table = await db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"}
+    )
+    ```
+
+=== "JavaScript"
+
+    <!-- skip-test -->
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect("s3://bucket/path");
+    const table = db.createTable(
+        "table",
+        [{ a: 1, b: 2}],
+        {storageOptions: {timeout: "60s"}}
+    );
+    ```
+
+!!! info "Storage option casing"
+
+    The storage option keys are case-insensitive. So `connect_timeout` and `CONNECT_TIMEOUT` are the same setting. Usually lowercase is used in the `storage_options` argument and uppercase is used for environment variables. In the `lancedb` Node package, the keys can also be provided in `camelCase` capitalization. For example, `connectTimeout` is equivalent to `connect_timeout`.
+
+### General configuration
+
+There are several options that can be set for all object stores, mostly related to network client configuration.
+
+<!-- from here: https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html -->
+
+| Key                        | Description                                                                                      |
+|----------------------------|--------------------------------------------------------------------------------------------------|
+| `allow_http`               | Allow non-TLS, i.e. non-HTTPS connections. Default: `False`.                                      |
+| `allow_invalid_certificates`| Skip certificate validation on HTTPS connections. Default: `False`.                               |
+| `connect_timeout`          | Timeout for only the connect phase of a Client. Default: `5s`.                                    |
+| `timeout`                  | Timeout for the entire request, from connection until the response body has finished. Default: `30s`. |
+| `user_agent`               | User agent string to use in requests.                                                             |
+| `proxy_url`                | URL of a proxy server to use for requests. Default: `None`.                                       |
+| `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
+| `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
+
+
+### AWS S3
+
+To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
+These can be set as environment variables or passed in the `storage_options` parameter:
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "aws_access_key_id": "my-access-key",
+            "aws_secret_access_key": "my-secret-key",
+            "aws_session_token": "my-session-token",
+        }
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://bucket/path",
+        {
+            storageOptions: {
+                awsAccessKeyId: "my-access-key",
+                awsSecretAccessKey: "my-secret-key",
+                awsSessionToken: "my-session-token",
+            }
+        }
+    );
+    ```
+
+Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
+
+The following keys can be used as both environment variables or keys in the `storage_options` parameter:
+
+| Key                                | Description                                                                                          |
+|------------------------------------|------------------------------------------------------------------------------------------------------|
+| `aws_region` / `region`             | The AWS region the bucket is in. This can be automatically detected when using AWS S3, but must be specified for S3-compatible stores. |
+| `aws_access_key_id` / `access_key_id` | The AWS access key ID to use.                                                                       |
+| `aws_secret_access_key` / `secret_access_key` | The AWS secret access key to use.                                                               |
+| `aws_session_token` / `session_token` | The AWS session token to use.                                                                     |
+| `aws_endpoint` / `endpoint`         | The endpoint to use for S3-compatible stores.                                                       |
+| `aws_virtual_hosted_style_request` / `virtual_hosted_style_request` | Whether to use virtual hosted-style requests, where the bucket name is part of the endpoint. Meant to be used with `aws_endpoint`. Default: `False`. |
+| `aws_s3_express` / `s3_express`     | Whether to use S3 Express One Zone endpoints. Default: `False`. See more details below.             |
+| `aws_server_side_encryption`        | The server-side encryption algorithm to use. Must be one of `"AES256"`, `"aws:kms"`, or `"aws:kms:dsse"`. Default: `None`. |
+| `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
+| `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
+
+
+!!! tip "Automatic cleanup for failed writes"
+
+    LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
+
+    **[Configuring a bucket lifecycle configuration to delete incomplete multipart uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html)**
+
+#### AWS IAM Permissions
+
+If a bucket is private, then an IAM policy must be specified to allow access to it. For many development scenarios, using broad permissions such as a PowerUser account is more than sufficient for working with LanceDB. However, in many production scenarios, you may wish to have as narrow as possible permissions.
+
+For **read and write access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:PutObject",
+              "s3:GetObject",
+              "s3:DeleteObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+For **read-only access**, LanceDB will need a policy such as:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+            ],
+            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": "arn:aws:s3:::<bucket>",
+            "Condition": {
+                "StringLike": {
+                    "s3:prefix": [
+                        "<prefix>/*"
+                    ]
+                }
+            }
+        }
+    ]
+}
+```
+
+#### S3-compatible stores
+
+LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        }
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://bucket/path",
+        {
+            storageOptions: {
+                region: "us-east-1",
+                endpoint: "http://minio:9000",
+            }
+        }
+    );
+    ```
+
+This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
+
+!!! tip "Local servers"
+
+    For local development, the server often has a `http` endpoint rather than a
+    secure `https` endpoint. In this case, you must also set the `ALLOW_HTTP`
+    environment variable to `true` to allow non-TLS connections, or pass the
+    storage option `allow_http` as `true`. If you do not do this, you will get
+    an error like `URL scheme is not allowed`.
+
+#### S3 Express
+
+LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
+
+To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        }
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        {
+            storageOptions: {
+                region: "us-east-1",
+                s3Express: "true",
+            }
+        }
+    );
+    ```
+
+
+### Google Cloud Storage
+
+GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. Alternatively, you can pass the path to the JSON file in the `storage_options`:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        }
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "gs://my-bucket/my-database",
+        {
+            storageOptions: {
+                serviceAccount: "path/to/service-account.json",
+            }
+        }
+    );
+    ```
+
+
+!!! info "HTTP/2 support"
+
+    By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
+
+
+The following keys can be used as both environment variables or keys in the `storage_options` parameter:
+<!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
+
+| Key                                   | Description                                  |
+|---------------------------------------|----------------------------------------------|
+| ``google_service_account`` / `service_account` | Path to the service account JSON file.       |
+| ``google_service_account_key``        | The serialized service account key.          |
+| ``google_application_credentials``    | Path to the application credentials.         |
+
+
+### Azure Blob Storage
+
+Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "az://my-container/my-database",
+        storage_options={
+            account_name: "some-account",
+            account_key: "some-key",
+        }
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+    const db = await lancedb.connect(
+        "az://my-container/my-database",
+        {
+            storageOptions: {
+                accountName: "some-account",
+                accountKey: "some-key",
+            }
+        }
+    );
+    ```
+
+These keys can be used as both environment variables or keys in the `storage_options` parameter:
+
+<!-- source: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html -->
+
+| Key                                   | Description                                                                                      |
+|---------------------------------------|--------------------------------------------------------------------------------------------------|
+| ``azure_storage_account_name``        | The name of the azure storage account.                                                           |
+| ``azure_storage_account_key``         | The serialized service account key.                                                              |
+| ``azure_client_id``                   | Service principal client id for authorizing requests.                                            |
+| ``azure_client_secret``               | Service principal client secret for authorizing requests.                                        |
+| ``azure_tenant_id``                   | Tenant id used in oauth flows.                                                                   |
+| ``azure_storage_sas_key``             | Shared access signature. The signature is expected to be percent-encoded, much like they are provided in the azure storage explorer or azure portal. |
+| ``azure_storage_token``               | Bearer token.                                                                                    |
+| ``azure_storage_use_emulator``        | Use object store with azurite storage emulator.                                                  |
+| ``azure_endpoint``                    | Override the endpoint used to communicate with blob storage.                                      |
+| ``azure_use_fabric_endpoint``         | Use object store with url scheme account.dfs.fabric.microsoft.com.                               |
+| ``azure_msi_endpoint``                | Endpoint to request a imds managed identity token.                                               |
+| ``azure_object_id``                   | Object id for use with managed identity authentication.                                          |
+| ``azure_msi_resource_id``             | Msi resource id for use with managed identity authentication.                                    |
+| ``azure_federated_token_file``        | File containing token for Azure AD workload identity federation.                                 |
+| ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
+| ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
+
+<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->