separate local and cloud connect

* Easy to type
* Handle `String, &str, [String] and [&str]` well without manual
conversion
* Fix function name to be verb
* Improve docstring of Rust.
* Promote `query` and `search()` to public `Table` trait

.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 0.4.4
+commit = True
+message = Bump version: {current_version} → {new_version}
+tag = True
+tag_name = v{new_version}
+
+[bumpversion:file:node/package.json]
+
+[bumpversion:file:rust/ffi/node/Cargo.toml]
+
+[bumpversion:file:rust/vectordb/Cargo.toml]

.bumpversion.toml

@@ -1,57 +0,0 @@
-[tool.bumpversion]
-current_version = "0.5.2"
-parse = """(?x)
-    (?P<major>0|[1-9]\\d*)\\.
-    (?P<minor>0|[1-9]\\d*)\\.
-    (?P<patch>0|[1-9]\\d*)
-    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
-"""
-serialize = [
-    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
-    "{major}.{minor}.{patch}",
-]
-search = "{current_version}"
-replace = "{new_version}"
-regex = false
-ignore_missing_version = false
-ignore_missing_files = false
-tag = true
-sign_tags = false
-tag_name = "v{new_version}"
-tag_message = "Bump version: {current_version} → {new_version}"
-allow_dirty = true
-commit = true
-message = "Bump version: {current_version} → {new_version}"
-commit_args = ""
-
-[tool.bumpversion.parts.pre_l]
-values = ["beta", "final"]
-optional_value = "final"
-
-[[tool.bumpversion.files]]
-filename = "node/package.json"
-search = "\"version\": \"{current_version}\","
-replace = "\"version\": \"{new_version}\","
-
-[[tool.bumpversion.files]]
-filename = "nodejs/package.json"
-search = "\"version\": \"{current_version}\","
-replace = "\"version\": \"{new_version}\","
-
-# nodejs binary packages
-[[tool.bumpversion.files]]
-glob = "nodejs/npm/*/package.json"
-search = "\"version\": \"{current_version}\","
-replace = "\"version\": \"{new_version}\","
-
-# Cargo files
-# ------------
-[[tool.bumpversion.files]]
-filename = "rust/ffi/node/Cargo.toml"
-search = "\nversion = \"{current_version}\""
-replace = "\nversion = \"{new_version}\""
-
-[[tool.bumpversion.files]]
-filename = "rust/lancedb/Cargo.toml"
-search = "\nversion = \"{current_version}\""
-replace = "\nversion = \"{new_version}\""

.cargo/config.toml

@@ -1,40 +0,0 @@
-[profile.release]
-lto = "fat"
-codegen-units = 1
-
-[profile.release-with-debug]
-inherits = "release"
-debug = true
-# Prioritize compile time over runtime performance
-codegen-units = 16
-lto = "thin"
-
-[target.'cfg(all())']
-rustflags = [
-    "-Wclippy::all",
-    "-Wclippy::style",
-    "-Wclippy::fallible_impl_from",
-    "-Wclippy::manual_let_else",
-    "-Wclippy::redundant_pub_crate",
-    "-Wclippy::string_add_assign",
-    "-Wclippy::string_add",
-    "-Wclippy::string_lit_as_bytes",
-    "-Wclippy::string_to_string",
-    "-Wclippy::use_self",
-    "-Dclippy::cargo",
-    "-Dclippy::dbg_macro",
-    # not too much we can do to avoid multiple crate versions
-    "-Aclippy::multiple-crate-versions",
-    "-Aclippy::wildcard_dependencies",
-]
-
-[target.x86_64-unknown-linux-gnu]
-rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
-
-[target.aarch64-apple-darwin]
-rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"]
-
-# Not all Windows systems have the C runtime installed, so this avoids library
-# not found errors on systems that are missing it.
-[target.x86_64-pc-windows-msvc]
-rustflags = ["-Ctarget-feature=+crt-static"]

.github/labeler.yml

@@ -1,33 +0,0 @@
-version: 1
-appendOnly: true
-# Labels are applied based on conventional commits standard
-# https://www.conventionalcommits.org/en/v1.0.0/
-# These labels are later used in release notes. See .github/release.yml
-labels:
-# If the PR title has an ! before the : it will be considered a breaking change
-# For example, `feat!: add new feature` will be considered a breaking change
-- label: breaking-change
-  title: "^[^:]+!:.*"
-- label: breaking-change
-  body: "BREAKING CHANGE"
-- label: enhancement
-  title: "^feat(\\(.+\\))?!?:.*"
-- label: bug
-  title: "^fix(\\(.+\\))?!?:.*"
-- label: documentation
-  title: "^docs(\\(.+\\))?!?:.*"
-- label: performance
-  title: "^perf(\\(.+\\))?!?:.*"
-- label: ci
-  title: "^ci(\\(.+\\))?!?:.*"
-- label: chore
-  title: "^(chore|test|build|style)(\\(.+\\))?!?:.*"
-- label: Python
-  files:
-    - "^python\\/.*"
-- label: Rust
-  files:
-    - "^rust\\/.*"
-- label: typescript
-  files:
-    - "^node\\/.*"

.github/release_notes.json

@@ -1,41 +0,0 @@
-{
-    "ignore_labels": ["chore"],
-    "pr_template": "- ${{TITLE}} by @${{AUTHOR}} in ${{URL}}",
-    "categories": [
-        {
-            "title": "## 🏆 Highlights",
-            "labels": ["highlight"]
-        },
-        {
-            "title": "## 🛠 Breaking Changes",
-            "labels": ["breaking-change"]
-        },
-        {
-            "title": "## ⚠️ Deprecations ",
-            "labels": ["deprecation"]
-        },
-        {
-            "title": "## 🎉 New Features",
-            "labels": ["enhancement"]
-        },
-        {
-            "title": "## 🐛 Bug Fixes",
-            "labels": ["bug"]
-        },
-        {
-            "title": "## 📚 Documentation",
-            "labels": ["documentation"]
-        },
-        {
-            "title": "## 🚀 Performance Improvements",
-            "labels": ["performance"]
-        },
-        {
-            "title": "## Other Changes"
-        },
-        {
-            "title": "## 🔧 Build and CI",
-            "labels": ["ci"]
-        }
-    ]
-}

.github/workflows/build_linux_wheel/action.yml

@@ -1,63 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build-linux-wheel
-description: "Build a manylinux wheel for lance"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11, 12"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-  arm-build:
-    description: "Build for arm64 instead of x86_64"
-    # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
-    required: false
-    default: "false"
-  manylinux:
-    description: "The manylinux version to build for"
-    required: false
-    default: "2_17"
-runs:
-  using: "composite"
-  steps:
-    - name: CONFIRM ARM BUILD
-      shell: bash
-      run: |
-        echo "ARM BUILD: ${{ inputs.arm-build }}"
-    - name: Build x86_64 Manylinux wheel
-      if: ${{ inputs.arm-build == 'false' }}
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        working-directory: python
-        target: x86_64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
-        args: ${{ inputs.args }}
-        before-script-linux: |
-          set -e
-          yum install -y openssl-devel \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
-    - name: Build Arm Manylinux Wheel
-      if: ${{ inputs.arm-build == 'true' }}
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        working-directory: python
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
-        target: aarch64-unknown-linux-gnu
-        manylinux: ${{ inputs.manylinux }}
-        args: ${{ inputs.args }}
-        before-script-linux: |
-          set -e
-          apt install -y unzip
-          if [ $(uname -m) = "x86_64" ]; then
-            PROTOC_ARCH="x86_64"
-          else
-            PROTOC_ARCH="aarch_64"
-          fi
-          curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$PROTOC_ARCH.zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip

.github/workflows/build_mac_wheel/action.yml

@@ -1,26 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build_wheel
-description: "Build a lance wheel"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-runs:
-  using: "composite"
-  steps:
-    - name: Install macos dependency
-      shell: bash
-      run: |
-        brew install protobuf
-    - name: Build wheel
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
-        working-directory: python
-        interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -1,34 +0,0 @@
-# We create a composite action to be re-used both for testing and for releasing
-name: build_wheel
-description: "Build a lance wheel"
-inputs:
-  python-minor-version:
-    description: "8, 9, 10, 11"
-    required: true
-  args:
-    description: "--release"
-    required: false
-    default: ""
-runs:
-  using: "composite"
-  steps:
-    - name: Install Protoc v21.12
-      working-directory: C:\
-      run: |
-        New-Item -Path 'C:\protoc' -ItemType Directory
-        Set-Location C:\protoc
-        Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-        7z x protoc.zip
-        Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-      shell: powershell
-    - name: Build wheel
-      uses: PyO3/maturin-action@v1
-      with:
-        command: build
-        args: ${{ inputs.args }}
-        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
-        working-directory: python
-    - uses: actions/upload-artifact@v3
-      with:
-        name: windows-wheels
-        path: python\target\wheels

.github/workflows/cargo-publish.yml

@@ -1,20 +1,13 @@
 name: Cargo Publish
 
 on:
-  push:
+  release:
-    tags-ignore:
+    types: [ published ]
-      # We don't publish pre-releases for Rust. Crates.io is just a source
-      # distribution, so we don't need to publish pre-releases.
-      - 'v*-beta*'
-      - '*-v*' # for example, python-vX.Y.Z
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Up-to-date compilers needed for fp16kernels.
-  CC: gcc-12
-  CXX: g++-12
 
 jobs:
   build:
@@ -23,7 +16,7 @@ jobs:
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -33,4 +26,4 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Publish the package
         run: |
-          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/dev.yml

@@ -1,81 +0,0 @@
-name: PR Checks
-
-on:
-  pull_request_target:
-    types: [opened, edited, synchronize, reopened]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  labeler:
-    permissions:
-      pull-requests: write
-    name: Label PR
-    runs-on: ubuntu-latest
-    steps:
-      - uses: srvaroa/labeler@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  commitlint:
-    permissions:
-      pull-requests: write
-    name: Verify PR title / description conforms to semantic-release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/setup-node@v3
-        with:
-          node-version: "18"
-      # These rules are disabled because Github will always ensure there
-      # is a blank line between the title and the body and Github will
-      # word wrap the description field to ensure a reasonable max line
-      # length.
-      - run: npm install @commitlint/config-conventional
-      - run: >
-          echo 'module.exports = {
-            "rules": {
-              "body-max-line-length": [0, "always", Infinity],
-              "footer-max-line-length": [0, "always", Infinity],
-              "body-leading-blank": [0, "always"]
-            }
-          }' > .commitlintrc.js
-      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
-        env:
-          COMMIT_MSG: >
-            ${{ github.event.pull_request.title }}
-
-            ${{ github.event.pull_request.body }}
-      - if: failure()
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const message = `**ACTION NEEDED**
-              
-              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
-
-              The PR title and description are used as the merge commit message.\
-              Please update your PR title and description to match the specification.
-
-              For details on the error please inspect the "PR Title Check" action.
-              `
-            // Get list of current comments
-            const comments = await github.paginate(github.rest.issues.listComments, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number
-            });
-            // Check if this job already commented
-            for (const comment of comments) {
-              if (comment.body === message) {
-                return // Already commented
-              }
-            }
-            // Post the comment about Conventional Commits
-            github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              body: message
-            })
-            core.setFailed(message)

.github/workflows/docs.yml

@@ -24,16 +24,12 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
-      - name: Install dependecies needed for ubuntu
-        run: |
-          sudo apt install -y protobuf-compiler libssl-dev
-          rustup update && rustup default        
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
           cache: "pip"
@@ -46,7 +42,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: 20
+          node-version: ${{ matrix.node-version }}
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - uses: Swatinem/rust-cache@v2
@@ -65,10 +61,10 @@ jobs:
         working-directory: node
         run: |
           npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
+          cp ../docs/src/javascript.md ../docs/src/javascript/javascript.md
       - name: Build docs
-        working-directory: docs
         run: |
-          PYTHONPATH=. mkdocs build
+          PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
       - name: Setup Pages
         uses: actions/configure-pages@v2
       - name: Upload artifact

.github/workflows/docs_test.yml

@@ -18,30 +18,26 @@ on:
 env:
   # Disable full debug symbol generation to speed up CI build and keep memory down
   # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
+  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
 
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
     steps:
     - name: Checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
-    - name: Install dependecies needed for ubuntu
-      run: |
-        sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.11
+        python-version: 3.${{ matrix.python-minor-version }}
         cache: "pip"
         cache-dependency-path: "docs/test/requirements.txt"
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
     - name: Build Python
       working-directory: docs/test
       run:
@@ -56,45 +52,45 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: ${{ matrix.os }}
-    timeout-minutes: 60
     strategy:
-      fail-fast: false
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
     steps:
     - name: Checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
-    - name: Print CPU capabilities
-      run: cat /proc/cpuinfo
     - name: Set up Node
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v3
       with:
-        node-version: 20
+        node-version: ${{ matrix.node-version }}
     - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
       run: |
         sudo apt install -y protobuf-compiler libssl-dev
-        rustup update && rustup default
-    - name: Rust cache
-      uses: swatinem/rust-cache@v2
     - name: Install node dependencies
       run: |
-        sudo swapoff -a
+        cd docs/test
-        sudo fallocate -l 8G /swapfile
+        npm install 
-        sudo chmod 600 /swapfile
+    - name: Rust cache
-        sudo mkswap /swapfile
+      uses: swatinem/rust-cache@v2
-        sudo swapon /swapfile
+    - name: Install LanceDB
-        sudo swapon --show
+      run: |
-        cd node
+        cd docs/test/node_modules/vectordb 
         npm ci
         npm run build-release
-        cd ../docs
+        npm run tsc
-        npm install
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
     - name: Test
       env:
         LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
         LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
       run: |
-        cd docs
+        cd docs/test/node
-        npm t
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/java.yml

@@ -1,85 +0,0 @@
-name: Build and Run Java JNI Tests
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - java/**
-      - rust/**
-      - .github/workflows/java.yml
-env:
-  # This env var is used by Swatinem/rust-cache@v2 for the cache
-  # key, so we set it to make sure it is always consistent.
-  CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
-  RUST_BACKTRACE: "1"
-  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
-  # CI builds are faster with incremental disabled.
-  CARGO_INCREMENTAL: "0"
-  CARGO_BUILD_JOBS: "1"
-jobs:
-  linux-build:
-    runs-on: ubuntu-22.04
-    name: ubuntu-22.04 + Java 11 & 17
-    defaults:
-      run:
-        working-directory: ./java
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: java/core/lancedb-jni
-      - name: Run cargo fmt
-        run: cargo fmt --check
-        working-directory: ./java/core/lancedb-jni
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Install Java 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 17
-          cache: "maven"
-      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
-      - name: Install Java 11
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 11
-          cache: "maven"
-      - name: Java Style Check
-        run: mvn checkstyle:check
-      # Disable because of issues in lancedb rust core code
-      # - name: Rust Clippy
-      #   working-directory: java/core/lancedb-jni
-      #   run: cargo clippy --all-targets -- -D warnings
-      - name: Running tests with Java 11
-        run: mvn clean test
-      - name: Running tests with Java 17
-        run: |
-          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
-          -XX:+IgnoreUnrecognizedVMOptions \
-          --add-opens=java.base/java.lang=ALL-UNNAMED \
-          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
-          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
-          --add-opens=java.base/java.io=ALL-UNNAMED \
-          --add-opens=java.base/java.net=ALL-UNNAMED \
-          --add-opens=java.base/java.nio=ALL-UNNAMED \
-          --add-opens=java.base/java.util=ALL-UNNAMED \
-          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
-          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
-          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
-          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
-          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
-          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
-          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
-          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
-          -Djdk.reflect.useDirectMethodHandle=false \
-          -Dio.netty.tryReflectionSetAccessible=true"
-          JAVA_HOME=$JAVA_17 mvn clean test

.github/workflows/make-release-commit.yml

@@ -1,99 +1,59 @@
 name: Create release commit
 
-# This workflow increments versions, tags the version, and pushes it.
-# When a tag is pushed, another workflow is triggered that creates a GH release
-# and uploads the binaries. This workflow is only for creating the tag.
-
-# This script will enforce that a minor version is incremented if there are any
-# breaking changes since the last minor increment. However, it isn't able to
-# differentiate between breaking changes in Node versus Python. If you wish to
-# bypass this check, you can manually increment the version and push the tag.
 on:
   workflow_dispatch:
     inputs:
       dry_run:
         description: 'Dry run (create the local commit/tags but do not push it)'
         required: true
-        default: false
+        default: "false"
-        type: boolean
-      type:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'preview'
         type: choice
         options:
-          - preview
+          - "true"
-          - stable
+          - "false"
-      python:
+      part:
-        description: 'Make a Python release'
+        description: 'What kind of release is this?'
         required: true
-        default: true
+        default: 'patch'
-        type: boolean
+        type: choice
-      other:
+        options:
-        description: 'Make a Node/Rust release'
+          - patch
-        required: true
+          - minor
-        default: true
+          - major
-        type: boolean
-      bump-minor:
-        description: 'Bump minor version'
-        required: true
-        default: false
-        type: boolean
 
 jobs:
-  make-release:
+  bump-version:
-    # Creates tag and GH release. The GH release will trigger the build and release jobs.
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
     steps:
-      - name: Output Inputs
+      - name: Check out main
-        run: echo "${{ toJSON(github.event.inputs) }}"
+        uses: actions/checkout@v3
-      - uses: actions/checkout@v4
         with:
+          ref: main
+          persist-credentials: false
           fetch-depth: 0
           lfs: true
-          # It's important we use our token here, as the default token will NOT
-          # trigger any workflows watching for new tags. See:
-          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
-          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
       - name: Set git configs for bumpversion
         shell: bash
         run: |
           git config user.name 'Lance Release'
           git config user.email 'lance-dev@lancedb.com'
-      - name: Set up Python 3.11
+      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.10"
-      - name: Bump Python version
+      - name: Bump version, create tag and commit
-        if: ${{ inputs.python }}
-        working-directory: python
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Need to get the commit before bumping the version, so we can
+          pip install bump2version
-          # determine if there are breaking changes in the next step as well.
+          bumpversion --verbose ${{ inputs.part }}
-          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
+      - name: Push new version and tag
-
+        if: ${{ inputs.dry_run }} == "false"
-          pip install bump-my-version PyGithub packaging
-          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
-      - name: Bump Node/Rust version
-        if: ${{ inputs.other }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          pip install bump-my-version PyGithub packaging
-          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
-      - name: Push new version tag
-        if: ${{ !inputs.dry_run }}
         uses: ad-m/github-push-action@master
         with:
-          # Need to use PAT here too to trigger next workflow. See comment above.
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-          branch: ${{ github.ref }}
+          branch: main
           tags: true
       - uses: ./.github/workflows/update_package_lock
-        if: ${{ !inputs.dry_run && inputs.other }}
+        if: ${{ inputs.dry_run }} == "false"
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+

.github/workflows/node.yml

@@ -20,11 +20,31 @@ env:
   # "1" means line tables only, which is useful for panic tracebacks.
   #
   # Use native CPU to accelerate tests if possible, especially for f16
-  # target-cpu=haswell fixes failing ci build
+  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=native -C target-feature=+f16c,+avx2,+fma"
-  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
   RUST_BACKTRACE: "1"
 
 jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: node/package-lock.json
+    - name: Lint
+      run: |
+        npm ci
+        npm run lint
   linux:
     name: Linux (Node ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -37,7 +57,7 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
@@ -69,7 +89,7 @@ jobs:
         shell: bash
         working-directory: node
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
@@ -107,9 +127,8 @@ jobs:
       AWS_ENDPOINT: http://localhost:4566
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
-      ALLOW_HTTP: true
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true

.github/workflows/nodejs.yml

@@ -28,12 +28,8 @@ jobs:
       run:
         shell: bash
         working-directory: nodejs
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
@@ -52,7 +48,7 @@ jobs:
         cargo fmt --all -- --check
         cargo clippy --all --all-features -- -D warnings
         npm ci
-        npm run lint-ci
+        npm run lint
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -65,7 +61,7 @@ jobs:
         shell: bash
         working-directory: nodejs
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
@@ -84,22 +80,17 @@ jobs:
       run: |
         npm ci
         npm run build
-    - name: Setup localstack
-      working-directory: .
-      run: docker compose up --detach --wait
     - name: Test
-      env:
-        S3_TEST: "1"
       run: npm run test
   macos:
     timeout-minutes: 30
-    runs-on: "macos-14"
+    runs-on: "macos-13"
     defaults:
       run:
         shell: bash
         working-directory: nodejs
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
@@ -120,3 +111,4 @@ jobs:
     - name: Test
       run: |
         npm run test
+

.github/workflows/npm-publish.yml

@@ -1,9 +1,8 @@
 name: NPM Publish
 
 on:
-  push:
+  release:
-    tags:
+    types: [ published ]
-      - "v*"
 
 jobs:
   node:
@@ -16,11 +15,11 @@ jobs:
         working-directory: node
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          cache: "npm"
+          cache: 'npm'
           cache-dependency-path: node/package-lock.json
       - name: Install dependencies
         run: |
@@ -32,7 +31,7 @@ jobs:
           npm run tsc
           npm pack
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: node-package
           path: |
@@ -46,13 +45,13 @@ jobs:
             runner: macos-13
           - arch: aarch64-apple-darwin
             # xlarge is implicitly arm64.
-            runner: macos-14
+            runner: macos-13-xlarge
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Install system dependencies
         run: brew install protobuf
       - name: Install npm dependencies
@@ -62,41 +61,12 @@ jobs:
       - name: Build MacOS native node modules
         run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-darwin-${{ matrix.config.arch }}
+          name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
          
-  nodejs-macos:
-    strategy:
-      matrix:
-        config:
-          - arch: x86_64-apple-darwin
-            runner: macos-13
-          - arch: aarch64-apple-darwin
-            # xlarge is implicitly arm64.
-            runner: macos-14
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build MacOS native nodejs modules
-        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-darwin-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
@@ -110,85 +80,20 @@ jobs:
           - arch: x86_64
             runner: ubuntu-latest
           - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-4vcpu-ubuntu-2204-arm
-            runner: warp-ubuntu-latest-arm64-4x
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
-      # To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
       - name: Build Linux Artifacts
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-linux-${{ matrix.config.arch }}
+          name: native-linux
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
-  nodejs-linux:
-    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ${{ matrix.config.runner }}
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - arch: x86_64
-            runner: ubuntu-latest
-          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
-      - name: Configure aarch64 build
-        if: ${{ matrix.config.arch == 'aarch64' }}
-        run: |
-          free -h
-          sudo fallocate -l 16G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
-          # print info
-          swapon --show
-          free -h
-      - name: Build Linux Artifacts
-        run: |
-          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
-      - name: Upload Linux Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-linux-${{ matrix.config.arch }}
-          path: |
-            nodejs/dist/*.node
-      # The generic files are the same in all distros so we just pick
-      # one to do the upload.
-      - name: Upload Generic Artifacts
-        if: ${{ matrix.config.arch == 'x86_64' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-dist
-          path: |
-            nodejs/dist/*
-            !nodejs/dist/*.node
-
   node-windows:
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
@@ -199,7 +104,7 @@ jobs:
         target: [x86_64-pc-windows-msvc]
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -216,132 +121,40 @@ jobs:
       - name: Build Windows native node modules
         run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
       - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: node-native-windows
+          name: native-windows
           path: |
             node/dist/lancedb-vectordb-win32*.tgz
 
-  nodejs-windows:
-    runs-on: windows-2022
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-pc-windows-msvc]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install Protoc v21.12
-        working-directory: C:\
-        run: |
-          New-Item -Path 'C:\protoc' -ItemType Directory
-          Set-Location C:\protoc
-          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-          7z x protoc.zip
-          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
-        shell: powershell
-      - name: Install npm dependencies
-        run: |
-          cd nodejs
-          npm ci
-      - name: Build Windows native node modules
-        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
-      - name: Upload Windows Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: nodejs-native-windows
-          path: |
-            nodejs/dist/*.node
-
   release:
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v3
-        with:
-          pattern: node-*
       - name: Display structure of downloaded files
         run: ls -R
       - uses: actions/setup-node@v3
         with:
           node-version: 20
-          registry-url: "https://registry.npmjs.org"
+          registry-url: 'https://registry.npmjs.org'
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
-          # Tag beta as "preview" instead of default "latest". See lancedb 
-          # npm publish step for more info.
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            PUBLISH_ARGS="--tag preview"
-          fi
-
           mv */*.tgz .
           for filename in *.tgz; do
-            npm publish $PUBLISH_ARGS $filename
+            npm publish $filename
           done
 
-  release-nodejs:
-    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
-    runs-on: ubuntu-latest
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    defaults:
-      run:
-        shell: bash
-        working-directory: nodejs
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: nodejs-dist
-          path: nodejs/dist
-      - uses: actions/download-artifact@v4
-        name: Download arch-specific binaries
-        with:
-          pattern: nodejs-*
-          path: nodejs/nodejs-artifacts
-          merge-multiple: true
-      - name: Display structure of downloaded files
-        run: find .
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          registry-url: "https://registry.npmjs.org"
-      - name: Install napi-rs
-        run: npm install -g @napi-rs/cli
-      - name: Prepare artifacts
-        run: npx napi artifacts -d nodejs-artifacts
-      - name: Display structure of staged files
-        run: find npm
-      - name: Publish to NPM
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        # By default, things are published to the latest tag. This is what is
-        # installed by default if the user does not specify a version. This is
-        # good for stable releases, but for pre-releases, we want to publish to
-        # the "preview" tag so they can install with `npm install lancedb@preview`.
-        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
-        run: |
-          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
-            npm publish --access public --tag preview
-          else
-            npm publish --access public
-          fi
-
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: main
           persist-credentials: false
@@ -349,87 +162,4 @@ jobs:
           lfs: true
       - uses: ./.github/workflows/update_package_lock
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-
-  update-package-lock-nodejs:
-    needs: [release-nodejs]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-  gh-release:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
-        run: |
-          set -e
-          echo "Extracting tag and version from $GITHUB_REF"
-          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
-            VERSION=${BASH_REMATCH[1]}
-            TAG=v$VERSION
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Release Notes
-        id: release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
-        with:
-          configuration: .github/release_notes.json
-          toTag: ${{ steps.extract_version.outputs.tag }}
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.release_notes.outputs.changelog }}

.github/workflows/pypi-publish.yml

@@ -1,160 +1,31 @@
 name: PyPI Publish
 
 on:
-  push:
+  release:
-    tags:
+    types: [ published ]
-      - 'python-v*'
 
 jobs:
-  linux:
+  publish:
-    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        config:
-          - platform: x86_64
-            manylinux: "2_17"
-            extra_args: ""
-          - platform: x86_64
-            manylinux: "2_28"
-            extra_args: "--features fp16kernels"
-          - platform: aarch64
-            manylinux: "2_24"
-            extra_args: ""
-          # We don't build fp16 kernels for aarch64, because it uses 
-          # cross compilation image, which doesn't have a new enough compiler.
-    runs-on: "ubuntu-22.04"
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - uses: ./.github/workflows/build_linux_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip ${{ matrix.config.extra_args }}"
-          arm-build: ${{ matrix.config.platform == 'aarch64' }}
-          manylinux: ${{ matrix.config.manylinux }}
-      - uses: ./.github/workflows/upload_wheel
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  mac:
-    timeout-minutes: 60
-    runs-on: ${{ matrix.config.runner }}
-    strategy:
-      matrix:
-        config:
-          - target: x86_64-apple-darwin
-            runner: macos-13
-          - target: aarch64-apple-darwin
-            runner: macos-14
-    env:
-      MACOSX_DEPLOYMENT_TARGET: 10.15
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12
-      - uses: ./.github/workflows/build_mac_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
-      - uses: ./.github/workflows/upload_wheel
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  windows:
-    timeout-minutes: 60
-    runs-on: windows-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - uses: ./.github/workflows/build_windows_wheel
-        with:
-          python-minor-version: 8
-          args: "--release --strip"
-          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
-      - uses: ./.github/workflows/upload_wheel
-        with:
-          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          fury_token: ${{ secrets.FURY_TOKEN }}
-  gh-release:
     runs-on: ubuntu-latest
-    permissions:
+    # Only runs on tags that matches the python-make-release action
-      contents: write
+    if: startsWith(github.ref, 'refs/tags/python-v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          fetch-depth: 0
+          python-version: "3.8"
-          lfs: true
+      - name: Build distribution
-      - name: Extract version
-        id: extract_version
-        env:
-          GITHUB_REF: ${{ github.ref }}
         run: |
-          set -e
+          ls -la
-          echo "Extracting tag and version from $GITHUB_REF"
+          pip install wheel setuptools --upgrade
-          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
+          python setup.py sdist bdist_wheel
-            VERSION=${BASH_REMATCH[1]}
+      - name: Publish
-            TAG=python-v$VERSION
+        uses: pypa/gh-action-pypi-publish@v1.8.5
-            echo "tag=$TAG" >> $GITHUB_OUTPUT
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
-          else
-            echo "Failed to extract version from $GITHUB_REF"
-            exit 1
-          fi
-          echo "Extracted version $VERSION from $GITHUB_REF"
-          if [[ $VERSION =~ beta ]]; then
-            echo "This is a beta release"
-
-            # Get last release (that is not this one)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          else
-            echo "This is a stable release"
-            # Get last stable tag (ignore betas)
-            FROM_TAG=$(git tag --sort='version:refname' \
-              | grep ^python-v \
-              | grep -vF "$TAG" \
-              | grep -v beta \
-              | python ci/semver_sort.py python-v \
-              | tail -n 1)
-          fi
-          echo "Found from tag $FROM_TAG"
-          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
-      - name: Create Python Release Notes
-        id: python_release_notes
-        uses: mikepenz/release-changelog-builder-action@v4
         with:
-          configuration: .github/release_notes.json
+          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          toTag: ${{ steps.extract_version.outputs.tag }}
+          packages-dir: python/dist
-          fromTag: ${{ steps.extract_version.outputs.from_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Create Python GH release
-        uses: softprops/action-gh-release@v2
-        with:
-          prerelease: ${{ contains('beta', github.ref) }}
-          tag_name: ${{ steps.extract_version.outputs.tag }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          generate_release_notes: false
-          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
-          body: ${{ steps.python_release_notes.outputs.changelog }}

.github/workflows/python-make-release-commit.yml

@@ -0,0 +1,56 @@
+name: Python - Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Set git configs for bumpversion
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Bump version, create tag and commit
+      working-directory: python
+      run: |
+        pip install bump2version
+        bumpversion --verbose ${{ inputs.part }}
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+        branch: main
+        tags: true
+

.github/workflows/python.yml

@@ -14,135 +14,49 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  lint:
+  linux:
-    name: "Lint"
     timeout-minutes: 30
+    strategy:
+      matrix:
+        python-minor-version: [ "8", "9", "10", "11" ]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-        uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
-          python-version: "3.11"
+        python-version: 3.${{ matrix.python-minor-version }}
-      - name: Install ruff
+    - name: Install lancedb
       run: |
-          pip install ruff==0.2.2
+        pip install -e .[tests]
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock ruff
     - name: Format check
       run: ruff format --check .
     - name: Lint
       run: ruff .
-  doctest:
+    - name: Run tests
-    name: "Doctest"
+      run: pytest -m "not slow" -x -v --durations=30 tests
-    timeout-minutes: 30
+    - name: doctest
-    runs-on: "ubuntu-22.04"
+      run: pytest --doctest-modules lancedb
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-          cache: "pip"
-      - name: Install protobuf
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
-      - name: Install
-        run: |
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
-          pip install tantivy
-          pip install mlx
-      - name: Doctest
-        run: pytest --doctest-modules python/lancedb
-  linux:
-    name: "Linux: python-3.${{ matrix.python-minor-version }}"
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        python-minor-version: ["9", "11"]
-    runs-on: "ubuntu-22.04"
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Install protobuf
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.${{ matrix.python-minor-version }}
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
-      - uses: ./.github/workflows/build_linux_wheel
-      - uses: ./.github/workflows/run_tests
-        with:
-          integration: true
-      # Make sure wheels are not included in the Rust cache
-      - name: Delete wheels
-        run: rm -rf target/wheels
   platform:
-    name: "Mac: ${{ matrix.config.name }}"
+    name: "Platform: ${{ matrix.config.name }}"
     timeout-minutes: 30
     strategy:
       matrix:
         config:
-          - name: x86
+          - name: x86 Mac
             runner: macos-13
-          - name: Arm
+          - name: Arm Mac
-            runner: macos-14
+            runner: macos-13-xlarge
-    runs-on: "${{ matrix.config.runner }}"
+          - name: x86 Windows
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
-      - uses: Swatinem/rust-cache@v2
-        with:
-          workspaces: python
-      - uses: ./.github/workflows/build_mac_wheel
-      - uses: ./.github/workflows/run_tests
-      # Make sure wheels are not included in the Rust cache
-      - name: Delete wheels
-        run: rm -rf target/wheels
-  windows:
-    name: "Windows: ${{ matrix.config.name }}"
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        config:
-          - name: x86
             runner: windows-latest
     runs-on: "${{ matrix.config.runner }}"
     defaults:
@@ -150,22 +64,21 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-        uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
         python-version: "3.11"
-      - uses: Swatinem/rust-cache@v2
+    - name: Install lancedb
-        with:
+      run: |
-          workspaces: python
+        pip install -e .[tests]
-      - uses: ./.github/workflows/build_windows_wheel
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-      - uses: ./.github/workflows/run_tests
+        pip install pytest pytest-mock
-      # Make sure wheels are not included in the Rust cache
+    - name: Run tests
-      - name: Delete wheels
+      run: pytest -m "not slow" -x -v --durations=30 tests
-        run: rm -rf target/wheels
   pydantic1x:
     timeout-minutes: 30
     runs-on: "ubuntu-22.04"
@@ -174,22 +87,21 @@ jobs:
         shell: bash
         working-directory: python
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         lfs: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler
     - name: Set up Python
-        uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install lancedb
       run: |
         pip install "pydantic<2"
-          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
+        pip install -e .[tests]
-          pip install tantivy
+        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install pytest pytest-mock
     - name: Run tests
-        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests
+      run: pytest -m "not slow" -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb

.github/workflows/run_tests/action.yml

@@ -1,31 +0,0 @@
-name: run-tests
-
-description: "Install lance wheel and run unit tests"
-inputs:
-  python-minor-version:
-    required: true
-    description: "8 9 10 11 12"
-  integration:
-    required: false
-    description: "Run integration tests"
-    default: "false"
-runs:
-  using: "composite"
-  steps:
-    - name: Install lancedb
-      shell: bash
-      run: |
-        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
-    - name: Setup localstack for integration tests
-      if: ${{ inputs.integration == 'true' }}
-      shell: bash
-      working-directory: .
-      run: docker compose up --detach --wait
-    - name: pytest (with integration)
-      shell: bash
-      if: ${{ inputs.integration == 'true' }}
-      run: pytest -m "not slow" -x -v --durations=30 python/python/tests
-    - name: pytest (no integration tests)
-      shell: bash
-      if: ${{ inputs.integration != 'true' }}
-      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -31,12 +31,8 @@ jobs:
       run:
         shell: bash
         working-directory: rust
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
           fetch-depth: 0
           lfs: true
@@ -58,12 +54,8 @@ jobs:
       run:
         shell: bash
         working-directory: rust
-    env:
-      # Need up-to-date compilers for kernels
-      CC: gcc-12
-      CXX: g++-12
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v3
       with:
           fetch-depth: 0
           lfs: true
@@ -74,27 +66,22 @@ jobs:
       run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
-    - name: Start S3 integration test environment
-      working-directory: .
-      run: docker compose up --detach --wait
     - name: Build
       run: cargo build --all-features
     - name: Run tests
       run: cargo test --all-features
-    - name: Run examples
-      run: cargo run --example simple
   macos:
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-14" ]
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
     runs-on: "${{ matrix.mac-runner }}"
     defaults:
       run:
         shell: bash
         working-directory: rust
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
         with:
             fetch-depth: 0
             lfs: true
@@ -108,12 +95,11 @@ jobs:
       - name: Build
         run: cargo build --all-features
       - name: Run tests
-        # Run with everything except the integration tests.
+        run: cargo test --all-features
-        run: cargo test --features remote,fp16kernels
   windows:
     runs-on: windows-2022
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: rust
@@ -131,4 +117,3 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
-  

.github/workflows/update_package_lock_nodejs/action.yml

@@ -1,33 +0,0 @@
-name: update_package_lock_nodejs
-description: "Update nodejs's package.lock"
-
-inputs:
-  github_token:
-    required: true
-    description: "github token for the repo"
-
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 20
-    - name: Set git configs
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Update package-lock.json file
-      working-directory: ./nodejs
-      run: |
-        npm install
-        git add package-lock.json
-        git commit -m "Updating package-lock.json"
-      shell: bash
-    - name: Push changes
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ inputs.github_token }}
-        branch: main
-        tags: true

.github/workflows/update_package_lock_run.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           ref: main
           persist-credentials: false

.github/workflows/update_package_lock_run_nodejs.yml

@@ -1,19 +0,0 @@
-name: Update NodeJs package-lock.json
-
-on:
-  workflow_dispatch:
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          persist-credentials: false
-          fetch-depth: 0
-          lfs: true
-      - uses: ./.github/workflows/update_package_lock_nodejs
-        with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/upload_wheel/action.yml

@@ -1,44 +0,0 @@
-name: upload-wheel
-
-description: "Upload wheels to Pypi"
-inputs:
-  pypi_token:
-    required: true
-    description: "release token for the repo"
-  fury_token:
-    required: true
-    description: "release token for the fury repo"
-
-runs:
-  using: "composite"
-  steps:
-  - name: Install dependencies
-    shell: bash
-    run: |
-      python -m pip install --upgrade pip
-      pip install twine
-  - name: Choose repo
-    shell: bash
-    id: choose_repo
-    run: |
-      if [ ${{ github.ref }} == "*beta*" ]; then
-        echo "repo=fury" >> $GITHUB_OUTPUT
-      else
-        echo "repo=pypi" >> $GITHUB_OUTPUT
-      fi
-  - name: Publish to PyPI
-    shell: bash
-    env:
-      FURY_TOKEN: ${{ inputs.fury_token }}
-      PYPI_TOKEN: ${{ inputs.pypi_token }}
-    run: |
-      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
-        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
-        echo "Uploading $WHEEL to Fury"
-        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
-      else
-        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
-          --username __token__ \
-          --password $PYPI_TOKEN \
-          target/wheels/lancedb-*.whl
-      fi

.gitignore

@@ -6,7 +6,7 @@
 venv
 
 .vscode
-.zed
+
 rust/target
 rust/Cargo.lock
 
@@ -22,11 +22,6 @@ python/dist
 
 **/.hypothesis
 
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
 ## Javascript
 *.node
 **/node_modules
@@ -34,12 +29,9 @@ python/dist
 node/dist
 node/examples/**/package-lock.json
 node/examples/**/dist
-nodejs/lancedb/native*
 dist
 
 ## Rust
 target
 
-**/sccache.log
-
 Cargo.lock

.pre-commit-config.yaml

@@ -5,17 +5,17 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+    -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.2.2
+    rev: v0.0.277
     hooks:
     - id: ruff
-- repo: local
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
-    - id: local-biome-check
+    - id: isort
-      name: biome check
+      name: isort (python)
-      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
-      language: system
-      types: [text]
-      files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*

Cargo.toml

@@ -1,51 +1,37 @@
 [workspace]
-members = [
+members = ["rust/ffi/node", "rust/vectordb", "nodejs"]
-    "rust/ffi/node",
-    "rust/lancedb",
-    "nodejs",
-    "python",
-    "java/core/lancedb-jni",
-]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
 edition = "2021"
-authors = ["LanceDB Devs <dev@lancedb.com>"]
+authors = ["Lance Devs <dev@lancedb.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/lancedb/lancedb"
-description = "Serverless, low-latency vector database for AI applications"
-keywords = ["lancedb", "lance", "database", "vector", "search"]
-categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.13.0", "features" = ["dynamodb"] }
+lance = { "version" = "=0.9.9", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.13.0" }
+lance-index = { "version" = "=0.9.9" }
-lance-linalg = { "version" = "=0.13.0" }
+lance-linalg = { "version" = "=0.9.9" }
-lance-testing = { "version" = "=0.13.0" }
+lance-testing = { "version" = "=0.9.9" }
-lance-datafusion = { "version" = "=0.13.0" }
 # Note that this one does not include pyarrow
-arrow = { version = "51.0", optional = false }
+arrow = { version = "49.0.0", optional = false }
-arrow-array = "51.0"
+arrow-array = "49.0"
-arrow-data = "51.0"
+arrow-data = "49.0"
-arrow-ipc = "51.0"
+arrow-ipc = "49.0"
-arrow-ord = "51.0"
+arrow-ord = "49.0"
-arrow-schema = "51.0"
+arrow-schema = "49.0"
-arrow-arith = "51.0"
+arrow-arith = "49.0"
-arrow-cast = "51.0"
+arrow-cast = "49.0"
 async-trait = "0"
-chrono = "0.4.35"
+chrono = "0.4.23"
-datafusion-physical-plan = "37.1"
+half = { "version" = "=2.3.1", default-features = false, features = [
-half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
 object_store = "0.9.0"
-pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"
 num-traits = "0.2"
-regex = "1.10"
-lazy_static = "1"

README.md

@@ -1,13 +1,13 @@
 <div align="center">
 <p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
+<img width="275" alt="LanceDB Logo" src="https://user-images.githubusercontent.com/917119/226205734-6063d87a-1ecc-45fe-85be-1dea6383a3d8.png">
 
-**Developer-friendly, database for multimodal AI**
+**Developer-friendly, serverless vector database for AI applications**
 
 <a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
 <a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/) 
+[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
 [![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
 [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
@@ -20,7 +20,7 @@
 
 <hr />
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -36,7 +36,7 @@ The key features of LanceDB include:
 
 * GPU support in building vector index(*).
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -51,19 +51,12 @@ npm install vectordb
 const lancedb = require('vectordb');
 const db = await lancedb.connect('data/sample-lancedb');
 
-const table = await db.createTable({
+const table = await db.createTable('vectors',
-  name: 'vectors',
+      [{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-  data:  [
+       { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }])
-    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
-  ]
-})
 
 const query = table.search([0.1, 0.3]).limit(2);
 const results = await query.execute();
-
-// You can also search for rows by specific criteria without involving a vector search.
-const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
 ```
 
 **Python**
@@ -83,5 +76,5 @@ result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
+* 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
 * 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -13,9 +13,7 @@ docker build \
     .
 popd
 
-# We turn on memory swap to avoid OOM killer
 docker run \
     -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
     lancedb-node-manylinux \
     bash ci/manylinux_node/build.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-ARCH=${1:-x86_64}
-
-# We pass down the current user so that when we later mount the local files
-# into the container, the files are accessible by the current user.
-pushd ci/manylinux_nodejs
-docker build \
-    -t lancedb-nodejs-manylinux \
-    --build-arg="ARCH=$ARCH" \
-    --build-arg="DOCKER_USER=$(id -u)" \
-    --progress=plain \
-    .
-popd
-
-# We turn on memory swap to avoid OOM killer
-docker run \
-    -v $(pwd):/io -w /io \
-    --memory-swap=-1 \
-    lancedb-nodejs-manylinux \
-    bash ci/manylinux_nodejs/build.sh $ARCH

ci/build_macos_artifacts_nodejs.sh

@@ -1,34 +0,0 @@
-# Builds the macOS artifacts (nodejs binaries).
-# Usage: ./ci/build_macos_artifacts_nodejs.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-set -e
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/lancedb
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd nodejs
-    echo "Building nodejs library for $1"
-    export RUST_TARGET=$1
-    npm run build-release
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/build_windows_artifacts_nodejs.ps1

@@ -1,41 +0,0 @@
-# Builds the Windows artifacts (nodejs binaries).
-# Usage:  .\ci\build_windows_artifacts_nodejs.ps1 [target]
-# Targets supported:
-# - x86_64-pc-windows-msvc
-# - i686-pc-windows-msvc
-
-function Prebuild-Rust {
-    param (
-        [string]$target
-    )
-
-    # Building here for the sake of easier debugging.
-    Push-Location -Path "rust/lancedb"
-    Write-Host "Building rust library for $target"
-    $env:RUST_BACKTRACE=1
-    cargo build --release --target $target
-    Pop-Location
-}
-
-function Build-NodeBinaries {
-    param (
-        [string]$target
-    )
-
-    Push-Location -Path "nodejs"
-    Write-Host "Building nodejs library for $target"
-    $env:RUST_TARGET=$target
-    npm run build-release
-    Pop-Location
-}
-
-$targets = $args[0]
-if (-not $targets) {
-    $targets = "x86_64-pc-windows-msvc"
-}
-
-Write-Host "Building artifacts for targets: $targets"
-foreach ($target in $targets) {
-    Prebuild-Rust $target
-    Build-NodeBinaries $target
-}

ci/bump_version.sh

@@ -1,51 +0,0 @@
-set -e
-
-RELEASE_TYPE=${1:-"stable"}
-BUMP_MINOR=${2:-false}
-TAG_PREFIX=${3:-"v"} # Such as "python-v"
-HEAD_SHA=${4:-$(git rev-parse HEAD)}
-
-readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-
-PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
-echo "Found previous tag $PREV_TAG"
-
-# Initially, we don't want to tag if we are doing stable, because we will bump
-# again later. See comment at end for why.
-if [[ "$RELEASE_TYPE" == 'stable' ]]; then 
-  BUMP_ARGS="--no-tag"
-fi
-
-# If last is stable and not bumping minor
-if [[ $PREV_TAG != *beta* ]]; then
-    if [[ "$BUMP_MINOR" != "false" ]]; then
-      # X.Y.Z -> X.(Y+1).0-beta.0
-      bump-my-version bump -vv $BUMP_ARGS minor
-    else
-      # X.Y.Z -> X.Y.(Z+1)-beta.0
-      bump-my-version bump -vv $BUMP_ARGS patch
-    fi
-else
-  if [[ "$BUMP_MINOR" != "false" ]]; then
-    # X.Y.Z-beta.N -> X.(Y+1).0-beta.0
-    bump-my-version bump -vv $BUMP_ARGS minor
-  else
-    # X.Y.Z-beta.N -> X.Y.Z-beta.(N+1)
-    bump-my-version bump -vv $BUMP_ARGS pre_n
-  fi
-fi
-
-# The above bump will always bump to a pre-release version. If we are releasing
-# a stable version, bump the pre-release level ("pre_l") to make it stable.
-if [[ $RELEASE_TYPE == 'stable' ]]; then
-  # X.Y.Z-beta.N -> X.Y.Z
-  bump-my-version bump -vv pre_l
-fi
-
-# Validate that we have incremented version appropriately for breaking changes
-NEW_TAG=$(git describe --tags --exact-match HEAD)
-NEW_VERSION=$(echo $NEW_TAG | sed "s/^$TAG_PREFIX//")
-LAST_STABLE_RELEASE=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | grep -v beta | grep -vF "$NEW_TAG" | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
-LAST_STABLE_VERSION=$(echo $LAST_STABLE_RELEASE | sed "s/^$TAG_PREFIX//")
-
-python $SELF_DIR/check_breaking_changes.py $LAST_STABLE_RELEASE $HEAD_SHA $LAST_STABLE_VERSION $NEW_VERSION

ci/check_breaking_changes.py

@@ -1,35 +0,0 @@
-"""
-Check whether there are any breaking changes in the PRs between the base and head commits.
-If there are, assert that we have incremented the minor version.
-"""
-import argparse
-import os
-from packaging.version import parse
-
-from github import Github
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("base")
-    parser.add_argument("head")
-    parser.add_argument("last_stable_version")
-    parser.add_argument("current_version")
-    args = parser.parse_args()
-
-    repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"])
-    commits = repo.compare(args.base, args.head).commits
-    prs = (pr for commit in commits for pr in commit.get_pulls())
-
-    for pr in prs:
-        if any(label.name == "breaking-change" for label in pr.labels):
-            print(f"Breaking change in PR: {pr.html_url}")
-            break
-    else:
-        print("No breaking changes found.")
-        exit(0)
-    
-    last_stable_version = parse(args.last_stable_version)
-    current_version = parse(args.current_version)
-    if current_version.minor <= last_stable_version.minor:
-        print("Minor version is not greater than the last stable version.")
-        exit(1)

ci/manylinux_nodejs/Dockerfile

@@ -1,31 +0,0 @@
-# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an
-# environment with a very old glibc, so that we are compatible with a wide
-# range of linux distributions.
-ARG ARCH=x86_64
-
-FROM quay.io/pypa/manylinux2014_${ARCH}
-
-ARG ARCH=x86_64
-ARG DOCKER_USER=default_user
-
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
-# Protobuf is also installed as root.
-COPY install_protobuf.sh install_protobuf.sh
-RUN ./install_protobuf.sh ${ARCH}
-
-ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
-
-# We switch to the user to install Rust and Node, since those like to be
-# installed at the user level.
-USER ${DOCKER_USER}
-
-COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
-RUN cp /prepare_manylinux_node.sh $HOME/ && \
-    cd $HOME && \
-    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/build.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Builds the nodejs module for manylinux. Invoked by ci/build_linux_artifacts_nodejs.sh.
-set -e
-ARCH=${1:-x86_64}
-
-if [ "$ARCH" = "x86_64" ]; then
-    export OPENSSL_LIB_DIR=/usr/local/lib64/
-else
-    export OPENSSL_LIB_DIR=/usr/local/lib/
-fi
-export OPENSSL_STATIC=1
-export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
-
-source $HOME/.bashrc
-
-cd nodejs
-npm ci
-npm run build-release

ci/manylinux_nodejs/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1u \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# Installs protobuf compiler. Should be run as root.
-set -e
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=x86_64
-else
-    # gnu target
-    ARCH=aarch_64
-fi
-
-PB_REL=https://github.com/protocolbuffers/protobuf/releases
-PB_VERSION=23.1
-curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-install_node() {
-    echo "Installing node..."
-
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-
-    source "$HOME"/.bashrc
-
-    nvm install --no-progress 16
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-install_node
-install_rust

ci/semver_sort.py

@@ -1,35 +0,0 @@
-"""
-Takes a list of semver strings and sorts them in ascending order.
-"""
-
-import sys
-from packaging.version import parse, InvalidVersion
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument("prefix", default="v")
-    args = parser.parse_args()
-
-    # Read the input from stdin
-    lines = sys.stdin.readlines()
-
-    # Parse the versions
-    versions = []
-    for line in lines:
-        line = line.strip()
-        try:
-            version_str = line.removeprefix(args.prefix)
-            version = parse(version_str)
-        except InvalidVersion:
-            # There are old tags that don't follow the semver format
-            print(f"Invalid version: {line}", file=sys.stderr)
-            continue
-        versions.append((line, version))
-
-    # Sort the versions
-    versions.sort(key=lambda x: x[1])
-
-    # Print the sorted versions as original strings
-    for line, _ in versions:
-        print(line)

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:3.3
+    image: localstack/localstack:0.14
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb,kms
+      - SERVICES=s3,dynamodb
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
+      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
       interval: 5s
       retries: 3
       start_period: 10s

dockerfiles/Dockerfile

@@ -1,27 +0,0 @@
-#Simple base dockerfile that supports basic dependencies required to run lance with FTS and Hybrid Search
-#Usage docker build -t lancedb:latest -f Dockerfile .
-FROM python:3.10-slim-buster
-
-# Install Rust
-RUN apt-get update && apt-get install -y curl build-essential && \
-  curl https://sh.rustup.rs -sSf | sh -s -- -y
-
-# Set the environment variable for Rust
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-# Install protobuf compiler
-RUN apt-get install -y protobuf-compiler && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
-RUN apt-get -y update &&\
-  apt-get -y upgrade && \
-  apt-get -y install git
-
-
-# Verify installations
-RUN python --version && \
-  rustc --version && \
-  protoc --version
-
-RUN pip install tantivy lancedb

docs/README.md

@@ -33,12 +33,3 @@ You can run a local server to test the docs prior to deployment by navigating to
 cd docs
 mkdocs serve
 ```
-
-### Run doctest for typescript example
-
-```bash
-cd lancedb/docs
-npm i
-npm run build
-npm run all
-```

docs/mkdocs.yml

@@ -27,6 +27,7 @@ theme:
     - content.tabs.link
     - content.action.edit
     - toc.follow
+    # - toc.integrate
     - navigation.top
     - navigation.tabs
     - navigation.tabs.sticky
@@ -46,11 +47,10 @@ plugins:
         paths: [../python]
         options:
           docstring_style: numpy
-            heading_level: 3
+          heading_level: 4
           show_source: true
           show_symbol_type_in_heading: true
           show_signature_annotations: true
-            show_root_heading: true
           members_order: source
         import:
           # for cross references
@@ -67,9 +67,7 @@ markdown_extensions:
     line_spans: __span
     pygments_lang_class: true
 - pymdownx.inlinehilite
-  - pymdownx.snippets:
+- pymdownx.snippets
-      base_path: ..
-      dedent_subsections: true
 - pymdownx.superfences
 - pymdownx.tabbed:
     alternate_style: true
@@ -90,30 +88,15 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
-          - Hybrid search:
-              - Overview: hybrid_search/hybrid_search.md
-              - Comparing Rerankers: hybrid_search/eval.md
-              - Airbnb financial data example: notebooks/hybrid_search.ipynb
-          - Reranking:
-              - Quickstart: reranking/index.md
-              - Cohere Reranker: reranking/cohere.md
-              - Linear Combination Reranker: reranking/linear_combination.md
-              - Cross Encoder Reranker: reranking/cross_encoder.md
-              - ColBERT Reranker: reranking/colbert.md
-              - OpenAI Reranker: reranking/openai.md
-              - Building Custom Rerankers: reranking/custom_reranker.md
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
-          - Sync -> Async Migration Guide: migration.md
-          - Tuning retrieval performance:
-              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-              - Reranking: guides/tuning_retrievers/2_reranking.md
   - 🧬 Managing embeddings:
     - Overview: embeddings/index.md
-          - Embedding functions: embeddings/embedding_functions.md
+    - Explicit management: embeddings/embedding_explicit.md
-          - Available models: embeddings/default_embedding_functions.md
+    - Implicit management: embeddings/embedding_functions.md
-          - User-defined embedding functions: embeddings/custom_embedding_function.md
+    - Available Functions: embeddings/default_embedding_functions.md
+    - Custom Embedding Functions: embeddings/api.md
     - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
     - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
   - 🔌 Integrations:
@@ -121,12 +104,9 @@ nav:
     - Pandas and PyArrow: python/pandas_and_pyarrow.md
     - Polars: python/polars_arrow.md
     - DuckDB: python/duckdb.md
-          - LangChain:
+    - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-            - LangChain 🔗: integrations/langchain.md
+    - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+    - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
-          - LlamaIndex 🦙:
-            - LlamaIndex docs: integrations/llamaIndex.md
-            - LlamaIndex demo:  https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
     - Pydantic: python/pydantic.md
     - Voxel51: integrations/voxel51.md
     - PromptTools: integrations/prompttools.md
@@ -145,19 +125,14 @@ nav:
       - Serverless Website Chatbot: examples/serverless_website_chatbot.md
       - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
       - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
-          - 🦀 Rust:
+  - 🔧 CLI & Config: cli_config.md
-              - Overview: examples/examples_rust.md
   - 💭 FAQs: faq.md
   - ⚙️ API reference:
     - 🐍 Python: python/python.md
-          - 👾 JavaScript (vectordb): javascript/modules.md
+    - 👾 JavaScript: javascript/javascript.md
-          - 👾 JavaScript (lancedb): javascript/modules.md
-          - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
   - ☁️ LanceDB Cloud:
     - Overview: cloud/index.md
-          - API reference:
+
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
 
 - Quick start: basic.md
 - Concepts:
@@ -170,30 +145,15 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
-      - Hybrid search:
-          - Overview: hybrid_search/hybrid_search.md
-          - Comparing Rerankers: hybrid_search/eval.md
-          - Airbnb financial data example: notebooks/hybrid_search.ipynb
-      - Reranking:
-          - Quickstart: reranking/index.md
-          - Cohere Reranker: reranking/cohere.md
-          - Linear Combination Reranker: reranking/linear_combination.md
-          - Cross Encoder Reranker: reranking/cross_encoder.md
-          - ColBERT Reranker: reranking/colbert.md
-          - OpenAI Reranker: reranking/openai.md
-          - Building Custom Rerankers: reranking/custom_reranker.md
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
-      - Sync -> Async Migration Guide: migration.md
-      - Tuning retrieval performance:
-          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
-          - Reranking: guides/tuning_retrievers/2_reranking.md
 - Managing Embeddings:
   - Overview: embeddings/index.md
-      - Embedding functions: embeddings/embedding_functions.md
+  - Explicit management: embeddings/embedding_explicit.md
-      - Available models: embeddings/default_embedding_functions.md
+  - Implicit management: embeddings/embedding_functions.md
-      - User-defined embedding functions: embeddings/custom_embedding_function.md
+  - Available Functions: embeddings/default_embedding_functions.md
+  - Custom Embedding Functions: embeddings/api.md
   - "Example: Multi-lingual semantic search": notebooks/multi_lingual_example.ipynb
   - "Example: MultiModal CLIP Embeddings": notebooks/DisappearingEmbeddingFunction.ipynb
 - Integrations:
@@ -201,49 +161,35 @@ nav:
   - Pandas and PyArrow: python/pandas_and_pyarrow.md
   - Polars: python/polars_arrow.md
   - DuckDB : python/duckdb.md
-      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
+  - LangChain 🦜️🔗↗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+  - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
   - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
   - Pydantic: python/pydantic.md
   - Voxel51: integrations/voxel51.md
   - PromptTools: integrations/prompttools.md
-  - Examples:
+- Python examples:
   - examples/index.md
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
   - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
+- Javascript examples:
+  - Overview: examples/examples_js.md
+  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
   - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
   - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - API reference:
-      - Overview: api_reference.md
   - Python: python/python.md
-      - Javascript (vectordb): javascript/modules.md
+  - Javascript: javascript/javascript.md
-      - Javascript (lancedb): js/modules.md
-      - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
 - LanceDB Cloud:
     - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
 
 extra_css:
   - styles/global.css
   - styles/extra.css
 
-extra_javascript:
-  - "extra_js/init_ask_ai_widget.js"
-
 extra:
   analytics:
     provider: google
     property: G-B7NFM40W74
-  social:
-    - icon: fontawesome/brands/github
-      link: https://github.com/lancedb/lancedb
-    - icon: fontawesome/brands/x-twitter
-      link: https://twitter.com/lancedb
-    - icon: fontawesome/brands/linkedin
-      link: https://www.linkedin.com/company/lancedb

docs/package-lock.json

@@ -1,132 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "lancedb-docs-test",
-      "version": "1.0.0",
-      "license": "Apache 2",
-      "dependencies": {
-        "apache-arrow": "file:../node/node_modules/apache-arrow",
-        "vectordb": "file:../node"
-      },
-      "devDependencies": {
-        "@types/node": "^20.11.8",
-        "typescript": "^5.3.3"
-      }
-    },
-    "../node": {
-      "name": "vectordb",
-      "version": "0.4.6",
-      "cpu": [
-        "x64",
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "os": [
-        "darwin",
-        "linux",
-        "win32"
-      ],
-      "dependencies": {
-        "@apache-arrow/ts": "^14.0.2",
-        "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^14.0.2",
-        "axios": "^1.4.0"
-      },
-      "devDependencies": {
-        "@neon-rs/cli": "^0.0.160",
-        "@types/chai": "^4.3.4",
-        "@types/chai-as-promised": "^7.1.5",
-        "@types/mocha": "^10.0.1",
-        "@types/node": "^18.16.2",
-        "@types/sinon": "^10.0.15",
-        "@types/temp": "^0.9.1",
-        "@types/uuid": "^9.0.3",
-        "@typescript-eslint/eslint-plugin": "^5.59.1",
-        "cargo-cp-artifact": "^0.1",
-        "chai": "^4.3.7",
-        "chai-as-promised": "^7.1.1",
-        "eslint": "^8.39.0",
-        "eslint-config-standard-with-typescript": "^34.0.1",
-        "eslint-plugin-import": "^2.26.0",
-        "eslint-plugin-n": "^15.7.0",
-        "eslint-plugin-promise": "^6.1.1",
-        "mocha": "^10.2.0",
-        "openai": "^4.24.1",
-        "sinon": "^15.1.0",
-        "temp": "^0.9.4",
-        "ts-node": "^10.9.1",
-        "ts-node-dev": "^2.0.0",
-        "typedoc": "^0.24.7",
-        "typedoc-plugin-markdown": "^3.15.3",
-        "typescript": "*",
-        "uuid": "^9.0.0"
-      },
-      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.6",
-        "@lancedb/vectordb-darwin-x64": "0.4.6",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.6",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.6",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.6"
-      }
-    },
-    "../node/node_modules/apache-arrow": {
-      "version": "14.0.2",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "@types/command-line-args": "5.2.0",
-        "@types/command-line-usage": "5.0.2",
-        "@types/node": "20.3.0",
-        "@types/pad-left": "2.1.1",
-        "command-line-args": "5.2.1",
-        "command-line-usage": "7.0.1",
-        "flatbuffers": "23.5.26",
-        "json-bignum": "^0.0.3",
-        "pad-left": "^2.1.0",
-        "tslib": "^2.5.3"
-      },
-      "bin": {
-        "arrow2csv": "bin/arrow2csv.js"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "20.11.8",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
-      "integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
-      "dev": true,
-      "dependencies": {
-        "undici-types": "~5.26.4"
-      }
-    },
-    "node_modules/apache-arrow": {
-      "resolved": "../node/node_modules/apache-arrow",
-      "link": true
-    },
-    "node_modules/typescript": {
-      "version": "5.3.3",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
-      "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
-      "dev": true,
-      "bin": {
-        "tsc": "bin/tsc",
-        "tsserver": "bin/tsserver"
-      },
-      "engines": {
-        "node": ">=14.17"
-      }
-    },
-    "node_modules/undici-types": {
-      "version": "5.26.5",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
-      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
-      "dev": true
-    },
-    "node_modules/vectordb": {
-      "resolved": "../node",
-      "link": true
-    }
-  }
-}

docs/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "lancedb-docs-test",
-  "version": "1.0.0",
-  "description": "auto-generated tests from doc",
-  "author": "dev@lancedb.com",
-  "license": "Apache 2",
-  "dependencies": {
-    "apache-arrow": "file:../node/node_modules/apache-arrow",
-    "vectordb": "file:../node"
-  },
-  "scripts": {
-    "build": "tsc -b && cd ../node && npm run build-release",
-    "example": "npm run build && node",
-    "test": "npm run build && ls dist/*.js | xargs -n 1 node"
-  },
-  "devDependencies": {
-    "@types/node": "^20.11.8",
-    "typescript": "^5.3.3"
-  }
-}

docs/src/ann_indexes.md

@@ -7,18 +7,26 @@ for brute-force scanning of the entire vector space.
 A vector index is faster but less accurate than exhaustive search (kNN or flat search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
-## Disk-based Index
+Currently, LanceDB does *not* automatically create the ANN index.
+LanceDB has optimized code for kNN as well. For many use-cases, datasets under 100K vectors won't require index creation at all.
+If you can live with <100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
 
-Lance provides an `IVF_PQ` disk-based index. It uses **Inverted File Index (IVF)** to first divide
+In the future we will look to automatically create and configure the ANN index as data comes in.
-the dataset into `N` partitions, and then applies **Product Quantization** to compress vectors in each partition.
+
-See the [indexing](concepts/index_ivfpq.md) concepts guide for more information on how this works.
+## Types of Index
+
+Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+
+* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+   and then use **Product Quantization** to compress vectors in each partition.
+* `DiskANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+   represent the nearest neighbors of each vector.
 
 ## Creating an IVF_PQ Index
 
 Lance supports `IVF_PQ` index type by default.
 
 === "Python"
-
      Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
 
      ```python
@@ -38,42 +46,25 @@ Lance supports `IVF_PQ` index type by default.
      tbl.create_index(num_partitions=256, num_sub_vectors=96)
      ```
 
-=== "Typescript"
+=== "Javascript"
+     ```javascript
+     const vectordb = require('vectordb')
+     const db = await vectordb.connect('data/sample-lancedb')
 
-    ```typescript
+     let data = []
-    --8<--- "docs/src/ann_indexes.ts:import"
+     for (let i = 0; i < 10_000; i++) {
-
+         data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
-    --8<-- "docs/src/ann_indexes.ts:ingest"
+     }
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
      ```
 
-=== "Rust"
+- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:create_index"
-    ```
-
-    IVF_PQ index parameters are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/index/vector/struct.IvfPqIndexBuilder.html).
-
-The following IVF_PQ paramters can be specified:
-
-- **distance_type**: The distance metric to use. By default it uses euclidean distance "`L2`".
 We also support "cosine" and "dot" distance as well.
-- **num_partitions**: The number of partitions in the index. The default is the square root
+- **num_partitions** (default: 256): The number of partitions of the index.
-  of the number of rows.
+- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-
+For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
-!!! note
+a single PQ code.
-
-    In the synchronous python SDK and node's `vectordb` the default is 256. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
-
-- **num_sub_vectors**: The number of sub-vectors (M) that will be created during Product Quantization (PQ).
-  For D dimensional vector, it will be divided into `M` subvectors with dimension `D/M`, each of which is replaced by
-  a single PQ code. The default is the dimension of the vector divided by 16.
-
-!!! note
-
-    In the synchronous python SDK and node's `vectordb` the default is currently 96. This default has
-    changed in the asynchronous python SDK and node's `lancedb`.
 
 <figure markdown>
   ![IVF PQ](./assets/ivf_pq.png)
@@ -87,7 +78,7 @@ Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being
 
 You can specify the GPU device to train IVF partitions via
 
-- **accelerator**: Specify to `cuda` or `mps` (on Apple Silicon) to enable GPU training.
+- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
 
 === "Linux"
 
@@ -101,7 +92,7 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-=== "MacOS"
+=== "Macos"
 
      <!-- skip-test -->
      ```python
@@ -113,11 +104,12 @@ You can specify the GPU device to train IVF partitions via
      )
      ```
 
-Troubleshooting:
+Trouble shootings:
 
-If you see `AssertionError: Torch not compiled with CUDA enabled`, you need to [install
+If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
 PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
 
+
 ## Querying an ANN Index
 
 Querying vector indexes is done via the [search](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.search) function.
@@ -135,7 +127,6 @@ There are a couple of parameters that can be used to fine-tune the search:
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
-
      ```python
      tbl.search(np.random.random((1536))) \
          .limit(2) \
@@ -143,43 +134,41 @@ There are a couple of parameters that can be used to fine-tune the search:
          .refine_factor(10) \
          .to_pandas()
      ```
-
+     ```
-    ```text
                                               vector       item       _distance
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
      ```
 
-=== "Typescript"
+=== "Javascript"
-
+     ```javascript
-    ```typescript
+     const results_1 = await table
-    --8<-- "docs/src/ann_indexes.ts:search1"
+         .search(Array(1536).fill(1.2))
+         .limit(2)
+         .nprobes(20)
+         .refineFactor(10)
+         .execute()
      ```
 
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/ivf_pq.rs:search1"
-    ```
-
-    Vector search options are more fully defined in the [crate docs](https://docs.rs/lancedb/latest/lancedb/query/struct.Query.html#method.nearest_to).
-
 The search will return the data requested in addition to the distance of each item.
 
+
 ### Filtering (where clause)
 
 You can further filter the elements returned by a search using a where clause.
 
 === "Python"
-
      ```python
      tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
      ```
 
-=== "Typescript"
+=== "Javascript"
-
      ```javascript
-     --8<-- "docs/src/ann_indexes.ts:search2"
+     const results_2 = await table
+         .search(Array(1536).fill(1.2))
+         .where("id != '1141'")
+         .limit(2)
+         .execute()
      ```
 
 ### Projections (select clause)
@@ -187,42 +176,34 @@ You can further filter the elements returned by a search using a where clause.
 You can select the columns returned by the query using a select clause.
 
 === "Python"
-
      ```python
      tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
      ```
-
+     ```
-
-     ```text
         vector                                             _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...
      ```
 
-=== "Typescript"
+=== "Javascript"
-
+     ```javascript
-     ```typescript
+     const results_3 = await table
-     --8<-- "docs/src/ann_indexes.ts:search3"
+         .search(Array(1536).fill(1.2))
+         .select(["id"])
+         .limit(2)
+         .execute()
      ```
 
 ## FAQ
 
-### Why do I need to manually create an index?
-
-Currently, LanceDB does _not_ automatically create the ANN index.
-LanceDB is well-optimized for kNN (exhaustive search) via a disk-based index. For many use-cases,
-datasets of the order of ~100K vectors don't require index creation. If you can live with up to
-100ms latency, skipping index creation is a simpler workflow while guaranteeing 100% recall.
-
 ### When is it necessary to create an ANN vector index?
 
-`LanceDB` comes out-of-the-box with highly optimized SIMD code for computing vector similarity.
+`LanceDB` has manually-tuned SIMD code for computing vector distances.
-In our benchmarks, computing distances for 100K pairs of 1K dimension vectors takes **less than 20ms**.
+In our benchmarks, computing 100K pairs of 1K dimension vectors takes **less than 20ms**.
-We observe that for small datasets (~100K rows) or for applications that can accept 100ms latency,
+For small datasets (< 100K rows) or applications that can accept 100ms latency, vector indices are usually not necessary.
-vector indices are usually not necessary.
 
-For large-scale or higher dimension vectors, it can beneficial to create vector index for performance.
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
 
 ### How big is my index, and how many memory will it take?
 

docs/src/ann_indexes.ts

@@ -1,53 +0,0 @@
-// --8<-- [start:import]
-import * as vectordb from "vectordb";
-// --8<-- [end:import]
-
-(async () => {
-  // --8<-- [start:ingest]
-  const db = await vectordb.connect("data/sample-lancedb");
-
-  let data = [];
-  for (let i = 0; i < 10_000; i++) {
-    data.push({
-      vector: Array(1536).fill(i),
-      id: `${i}`,
-      content: "",
-      longId: `${i}`,
-    });
-  }
-  const table = await db.createTable("my_vectors", data);
-  await table.createIndex({
-    type: "ivf_pq",
-    column: "vector",
-    num_partitions: 16,
-    num_sub_vectors: 48,
-  });
-  // --8<-- [end:ingest]
-
-  // --8<-- [start:search1]
-  const results_1 = await table
-    .search(Array(1536).fill(1.2))
-    .limit(2)
-    .nprobes(20)
-    .refineFactor(10)
-    .execute();
-  // --8<-- [end:search1]
-
-  // --8<-- [start:search2]
-  const results_2 = await table
-    .search(Array(1536).fill(1.2))
-    .where("id != '1141'")
-    .limit(2)
-    .execute();
-  // --8<-- [end:search2]
-
-  // --8<-- [start:search3]
-  const results_3 = await table
-    .search(Array(1536).fill(1.2))
-    .select(["id"])
-    .limit(2)
-    .execute();
-  // --8<-- [end:search3]
-
-  console.log("Ann indexes: done");
-})();

docs/src/api_reference.md

@@ -1,8 +0,0 @@
-# API Reference
-
-The API reference for the LanceDB client SDKs are available at the following locations:
-
-- [Python](python/python.md)
-- [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/modules.md)
-- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -3,7 +3,7 @@
 !!! info "LanceDB can be run in a number of ways:"
 
     * Embedded within an existing backend (like your Django, Flask, Node.js or FastAPI application)
-    * Directly from a client application like a Jupyter notebook for analytical workloads
+    * Connected to directly from a client application like a Jupyter notebook for analytical workloads
     * Deployed as a remote serverless database
 
 ![](assets/lancedb_embedded_explanation.png)
@@ -11,134 +11,47 @@
 ## Installation
 
 === "Python"
-
       ```shell
       pip install lancedb
       ```
 
-=== "Typescript"
+=== "Javascript"
-
       ```shell
       npm install vectordb
       ```
 
-=== "Rust"
+## How to connect to a database
-
-    ```shell
-    cargo add lancedb
-    ```
-
-    !!! info "To use the lancedb create, you first need to install protobuf."
-
-    === "macOS"
-
-        ```shell
-        brew install protobuf
-        ```
-
-    === "Ubuntu/Debian"
-
-        ```shell
-        sudo apt install -y protobuf-compiler libssl-dev
-        ```
-
-    !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
-
-### Preview releases
-
-Stable releases are created about every 2 weeks. For the latest features and bug
-fixes, you can install the preview release. These releases receive the same
-level of testing as stable releases, but are not guaranteed to be available for
-more than 6 months after they are released. Once your application is stable, we
-recommend switching to stable releases.
 
 === "Python"
-
-      ```shell
-      pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
-      ```
-
-=== "Typescript"
-
-      ```shell
-      npm install vectordb@preview
-      ```
-
-=== "Rust"
-    
-    We don't push preview releases to crates.io, but you can referent the tag
-    in GitHub within your Cargo dependencies:
-
-    ```toml
-    [dependencies]
-    lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" }
-    ```
-
-## Connect to a database
-
-=== "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:imports"
+      import lancedb
-    --8<-- "python/python/tests/docs/test_basic.py:connect"
+      uri = "data/sample-lancedb"
-
+      db = lancedb.connect(uri)
-    --8<-- "python/python/tests/docs/test_basic.py:connect_async"
       ```
 
-    !!! note "Asynchronous Python API"
+      LanceDB will create the directory if it doesn't exist (including parent directories).
 
-        The asynchronous Python API is new and has some slight differences compared
+      If you need a reminder of the uri, use the `db.uri` property.
-        to the synchronous API.  Feel free to start using the asynchronous version.
-        Once all features have migrated we will start to move the synchronous API to
-        use the same syntax as the asynchronous API.  To help with this migration we
-        have created a [migration guide](migration.md) detailing the differences.
 
-=== "Typescript"
+=== "Javascript"
+      ```javascript
+      const lancedb = require("vectordb");
 
-    ```typescript
+      const uri = "data/sample-lancedb";
-    --8<-- "docs/src/basic_legacy.ts:import"
+      const db = await lancedb.connect(uri);
-
-    --8<-- "docs/src/basic_legacy.ts:open_db"
       ```
       
-    !!! note "`@lancedb/lancedb` vs. `vectordb`"
-
-        The Javascript SDK was originally released as `vectordb`.  In an effort to
-        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
-        API is being released as `lancedb`.  If you are starting new work we encourage
-        you to try out `lancedb`.  Once the new API is feature complete we will begin
-        slowly deprecating `vectordb` in favor of `lancedb`. There is a
-        [migration guide](migration.md) detailing the differences which will assist
-        you in this process.
-
-=== "Rust"
-
-    ```rust
-    #[tokio::main]
-    async fn main() -> Result<()> {
-        --8<-- "rust/lancedb/examples/simple.rs:connect"
-    }
-    ```
-
-    !!! info "See [examples/simple.rs](https://github.com/lancedb/lancedb/tree/main/rust/lancedb/examples/simple.rs) for a full working example."
-
       LanceDB will create the directory if it doesn't exist (including parent directories).
 
       If you need a reminder of the uri, you can call `db.uri()`.
 
-## Create a table
+## How to create a table
-
-### Create a table from initial data
-
-If you have data to insert into the table at creation time, you can simultaneously create a
-table and insert the data into it. The schema of the data will be used as the schema of the
-table.
 
 === "Python"
-
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table"
+    tbl = db.create_table("my_table",
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async"
+                    data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                          {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     If the table already exists, LanceDB will raise an error by default.
@@ -146,233 +59,126 @@ table.
     to the `create_table` method.
 
     You can also pass in a pandas DataFrame directly:
-
     ```python
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_pandas"
+    import pandas as pd
-    --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
+    df = pd.DataFrame([{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    tbl = db.create_table("table_from_df", data=df)
     ```
 
-=== "Typescript"
+=== "Javascript"
-
+    ```javascript
-    ```typescript
+    const tb = await db.createTable(
-    --8<-- "docs/src/basic_legacy.ts:create_table"
+        "myTable",
+        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}]
+    )
     ```
 
     If the table already exists, LanceDB will raise an error by default.
     If you want to overwrite the table, you can pass in `mode="overwrite"`
     to the `createTable` function.
 
-=== "Rust"
 
-    ```rust
+!!! info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
-    --8<-- "rust/lancedb/examples/simple.rs:create_table"
-    ```
 
-    If the table already exists, LanceDB will raise an error by default.  See
+### Creating an empty table
-    [the mode option](https://docs.rs/lancedb/latest/lancedb/connection/struct.CreateTableBuilder.html#method.mode)
-    for details on how to overwrite (or open) existing tables instead.
-
-    !!! Providing table records in Rust
-
-        The Rust SDK currently expects data to be provided as an Arrow
-        [RecordBatchReader](https://docs.rs/arrow-array/latest/arrow_array/trait.RecordBatchReader.html)
-        Support for additional formats (such as serde or polars) is on the roadmap.
-
-!!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
-
-!!! info "Automatic embedding generation with Embedding API"
-    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
-
-### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
-In this case, you can create an empty table and specify the schema, so that you can add
+In this case, you can create an empty table and specify the schema.
-data to the table at a later time (as long as it conforms to the schema). This is
-similar to a `CREATE TABLE` statement in SQL.
 
 === "Python"
-
       ```python
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table"
+      import pyarrow as pa
-      --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
+      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
+      tbl = db.create_table("empty_table", schema=schema)
       ```
 
-    !!! note "You can define schema in Pydantic"
+## How to open an existing table
-        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
 
-=== "Typescript"
+Once created, you can open a table using the following code:
-
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_empty_table"
-    ```
-
-## Open an existing table
-
-Once created, you can open a table as follows:
 
 === "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:open_table"
+      tbl = db.open_table("my_table")
-    --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
-    ```
-
-=== "Typescript"
-
-    ```typescript
-    const tbl = await db.openTable("myTable");
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:open_existing_tbl"
       ```
 
       If you forget the name of your table, you can always get a listing of all table names:
 
-=== "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:table_names"
+      print(db.table_names())
-    --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
       ```
 
 === "Javascript"
+      ```javascript
+      const tbl = await db.openTable("myTable");
+      ```
+
+      If you forget the name of your table, you can always get a listing of all table names:
 
       ```javascript
       console.log(await db.tableNames());
       ```
 
-=== "Rust"
+## How to add data to a table
 
-    ```rust
+After a table has been created, you can always add more data to it using
-    --8<-- "rust/lancedb/examples/simple.rs:list_names"
-    ```
-
-## Add data to a table
-
-After a table has been created, you can always add more data to it as follows:
 
 === "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:add_data"
+
-    --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
+      # Option 1: Add a list of dicts to a table
+      data = [{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+              {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}]
+      tbl.add(data)
+
+      # Option 2: Add a pandas DataFrame to a table
+      df = pd.DataFrame(data)
+      tbl.add(data)
       ```
 
-=== "Typescript"
+=== "Javascript"
-
+      ```javascript
-    ```typescript
+      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
-    --8<-- "docs/src/basic_legacy.ts:add"
+                     {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
       ```
 
-=== "Rust"
+## How to search for (approximate) nearest neighbors
 
-    ```rust
+Once you've embedded the query, you can find its nearest neighbors using the following code:
-    --8<-- "rust/lancedb/examples/simple.rs:add"
-    ```
-
-## Search for nearest neighbors
-
-Once you've embedded the query, you can find its nearest neighbors as follows:
 
 === "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search"
+      tbl.search([100, 100]).limit(2).to_pandas()
-    --8<-- "python/python/tests/docs/test_basic.py:vector_search_async"
       ```
 
       This returns a pandas DataFrame with the results.
 
-=== "Typescript"
+=== "Javascript"
-
+      ```javascript
-    ```typescript
+      const query = await tbl.search([100, 100]).limit(2).execute();
-    --8<-- "docs/src/basic_legacy.ts:search"
       ```
 
-=== "Rust"
+## How to delete rows from a table
-
-    ```rust
-    use futures::TryStreamExt;
-
-    --8<-- "rust/lancedb/examples/simple.rs:search"
-    ```
-
-    !!! Query vectors in Rust
-        Rust does not yet support automatic execution of embedding functions.  You will need to
-        calculate embeddings yourself.  Support for this is on the roadmap and can be tracked at
-        https://github.com/lancedb/lancedb/issues/994
-
-        Query vectors can be provided as Arrow arrays or a Vec/slice of Rust floats.
-        Support for additional formats (e.g. `polars::series::Series`) is on the roadmap.
-
-By default, LanceDB runs a brute-force scan over dataset to find the K nearest neighbours (KNN).
-For tables with more than 50K vectors, creating an ANN index is recommended to speed up search performance.
-LanceDB allows you to create an ANN index on a table as follows:
-
-=== "Python"
-
-    ```py
-    --8<-- "python/python/tests/docs/test_basic.py:create_index"
-    --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
-    ```
-
-=== "Typescript"
-
-    ```{.typescript .ignore}
-    --8<-- "docs/src/basic_legacy.ts:create_index"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:create_index"
-    ```
-
-!!! note "Why do I need to create an index manually?"
-LanceDB does not automatically create the ANN index for two reasons. The first is that it's optimized
-for really fast retrievals via a disk-based index, and the second is that data and query workloads can
-be very diverse, so there's no one-size-fits-all index configuration. LanceDB provides many parameters
-to fine-tune index size, query latency and accuracy. See the section on
-[ANN indexes](ann_indexes.md) for more details.
-
-## Delete rows from a table
 
 Use the `delete()` method on tables to delete rows from a table. To choose
 which rows to delete, provide a filter that matches on the metadata columns.
 This can delete any number of rows that match the filter.
 
 === "Python"
-
       ```python
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows"
+      tbl.delete('item = "fizz"')
-    --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
       ```
 
-=== "Typescript"
+=== "Javascript"
-
+      ```javascript
-    ```typescript
+      await tbl.delete('item = "fizz"')
-    --8<-- "docs/src/basic_legacy.ts:delete"
-    ```
-
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:delete"
       ```
 
 The deletion predicate is a SQL expression that supports the same expressions
-as the `where()` clause (`only_if()` in Rust) on a search. They can be as
+as the `where()` clause on a search. They can be as simple or complex as needed.
-simple or complex as needed. To see what expressions are supported, see the
+To see what expressions are supported, see the [SQL filters](sql.md) section.
-[SQL filters](sql.md) section.
+
 
 === "Python"
 
@@ -382,40 +188,27 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
-=== "Rust"
+## How to remove a table
-
-      Read more: [lancedb::Table::delete](https://docs.rs/lancedb/latest/lancedb/table/struct.Table.html#method.delete)
-
-## Drop a table
 
 Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
-
       ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      db.drop_table("my_table")
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
       By default, if the table does not exist an exception is raised. To suppress this,
       you can pass in `ignore_missing=True`.
 
-=== "Typescript"
+=== "JavaScript"
-
+      ```javascript
-      ```typescript
+      await db.dropTable('myTable')
-      --8<-- "docs/src/basic_legacy.ts:drop_table"
       ```
 
       This permanently removes the table and is not recoverable, unlike deleting rows.
       If the table does not exist an exception is raised. 
 
-=== "Rust"
-
-    ```rust
-    --8<-- "rust/lancedb/examples/simple.rs:drop_table"
-    ```
-
 !!! note "Bundling `vectordb` apps with Webpack"
 
     If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
@@ -430,19 +223,6 @@ Use the `drop_table()` method on the database to remove a table.
     })
     ```
 
-## Using the Embedding API
-You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
-
-=== "Python"
-
-    ```python
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
-    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
-    ```
-
-Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
-
-
 ## What's next
 
 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.

docs/src/basic_legacy.ts

@@ -1,92 +0,0 @@
-// --8<-- [start:import]
-import * as lancedb from "vectordb";
-import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
-// --8<-- [end:import]
-import * as fs from "fs";
-import { Table as ArrowTable, Utf8 } from "apache-arrow";
-
-const example = async () => {
-  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
-  // --8<-- [start:open_db]
-  const lancedb = require("vectordb");
-  const uri = "data/sample-lancedb";
-  const db = await lancedb.connect(uri);
-  // --8<-- [end:open_db]
-
-  // --8<-- [start:create_table]
-  const tbl = await db.createTable(
-    "myTable",
-    [
-      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
-      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
-    ],
-    { writeMode: lancedb.WriteMode.Overwrite }
-  );
-  // --8<-- [end:create_table]
-
-  // --8<-- [start:add]
-  const newData = Array.from({ length: 500 }, (_, i) => ({
-    vector: [i, i + 1],
-    item: "fizz",
-    price: i * 0.1,
-  }));
-  await tbl.add(newData);
-  // --8<-- [end:add]
-
-  // --8<-- [start:create_index]
-  await tbl.createIndex({
-    type: "ivf_pq",
-    num_partitions: 2,
-    num_sub_vectors: 2,
-  });
-  // --8<-- [end:create_index]
-
-  // --8<-- [start:create_empty_table]
-  const schema = new Schema([
-    new Field("id", new Int32()),
-    new Field("name", new Utf8()),
-  ]);
-  const empty_tbl = await db.createTable({ name: "empty_table", schema });
-  // --8<-- [end:create_empty_table]
-
-  // --8<-- [start:create_f16_table]
-  const dim = 16
-  const total = 10
-  const f16_schema = new Schema([
-      new Field('id', new Int32()),
-      new Field(
-        'vector',
-        new FixedSizeList(dim, new Field('item', new Float16(), true)),
-        false
-      )
-    ])
-  const data = lancedb.makeArrowTable(
-      Array.from(Array(total), (_, i) => ({
-        id: i,
-        vector: Array.from(Array(dim), Math.random)
-      })),
-      { f16_schema }
-    )
-  const table = await db.createTable('f16_tbl', data)
-  // --8<-- [end:create_f16_table]
-
-  // --8<-- [start:search]
-  const query = await tbl.search([100, 100]).limit(2).execute();
-  // --8<-- [end:search]
-  console.log(query);
-
-  // --8<-- [start:delete]
-  await tbl.delete('item = "fizz"');
-  // --8<-- [end:delete]
-
-  // --8<-- [start:drop_table]
-  await db.dropTable("myTable");
-  // --8<-- [end:drop_table]
-};
-
-async function main() {
-  await example();
-  console.log("Basic example: done");
-}
-
-main();

docs/src/cli_config.md

@@ -0,0 +1,51 @@
+
+# CLI & Config
+
+## LanceDB CLI
+Once lanceDB is installed, you can access the CLI using `lancedb` command on the console.
+
+```
+lancedb
+```
+
+This lists out all the various command-line options available. You can get the usage or help for a particular command.
+
+```
+lancedb {command} --help
+```
+
+## LanceDB config
+LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
+To view your config settings, you can use:
+
+```
+lancedb config
+```
+
+These config parameters can be tuned using the cli.
+
+```
+lancedb {config_name} --{argument}
+```
+
+## LanceDB Opt-in Diagnostics
+When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
+These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
+
+### Get usage help
+
+```
+lancedb diagnostics --help
+```
+
+### Disable diagnostics
+
+```
+lancedb diagnostics --disabled
+```
+
+### Enable diagnostics
+
+```
+lancedb diagnostics --enabled
+```

docs/src/concepts/index_ivfpq.md

@@ -31,7 +31,7 @@ As an example, consider starting with 128-dimensional vector consisting of 32-bi
 
 While PQ helps with reducing the size of the index, IVF primarily addresses search performance. The primary purpose of an inverted file index is to facilitate rapid and effective nearest neighbor search by narrowing down the search space.
 
-In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are initialized by running K-means over the stored vectors. The centroids of K-means turn into the seed points which then each define a region. These regions are then are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
+In IVF, the PQ vector space is divided into *Voronoi cells*, which are essentially partitions that consist of all the points in the space that are within a threshold distance of the given region's seed point. These seed points are used to create an inverted index that correlates each centroid with a list of vectors in the space, allowing a search to be restricted to just a subset of vectors in the index.
 
 ![](../assets/ivfpq_ivf_desc.webp)
 
@@ -81,4 +81,24 @@ The above query will perform a search on the table `tbl` using the given query v
 * `to_pandas()`: Convert the results to a pandas DataFrame
 
 And there you have it! You now understand what an IVF-PQ index is, and how to create and query it in LanceDB.
-To see how to create an IVF-PQ index in LanceDB, take a look at the [ANN indexes](../ann_indexes.md) section.
+
+
+## FAQ
+
+### When is it necessary to create a vector index?
+
+LanceDB has manually-tuned SIMD code for computing vector distances. In our benchmarks, computing 100K pairs of 1K dimension vectors takes **<20ms**. For small datasets (<100K rows) or applications that can accept up to 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how much memory will it take?
+
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with 1024-dimension vectors, if we choose `num_sub_vectors = 64`, each sub-vector has `1024 / 64 = 16` float32 numbers. Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for IVF_PQ index?
+
+`num_partitions` is used to decide how many partitions the first level IVF index uses. Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train. On SIFT-1M dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency/recall.
+
+`num_sub_vectors` specifies how many PQ short codes to generate on each vector. Because PQ is a lossy compression of the original vector, a higher `num_sub_vectors` usually results in less space distortion, and thus yields better accuracy. However, a higher `num_sub_vectors` also causes heavier I/O and more PQ computation, and thus, higher latency. `dimension / num_sub_vectors` should be a multiple of 8 for optimum SIMD efficiency.

docs/src/embeddings/api.md

@@ -17,7 +17,6 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 
 ```python
 from lancedb.embeddings import register
-from lancedb.util import attempt_import_or_raise
 
 @register("sentence-transformers")
 class SentenceTransformerEmbeddings(TextEmbeddingFunction):
@@ -82,7 +81,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        open_clip = self.safe_import("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
         model, _, preprocess = open_clip.create_model_and_transforms(
             self.name, pretrained=self.pretrained
         )
@@ -110,14 +109,14 @@ class OpenClipEmbeddings(EmbeddingFunction):
         if isinstance(query, str):
             return [self.generate_text_embeddings(query)]
         else:
-            PIL = attempt_import_or_raise("PIL", "pillow")
+            PIL = self.safe_import("PIL", "pillow")
             if isinstance(query, PIL.Image.Image):
                 return [self.generate_image_embedding(query)]
             else:
                 raise TypeError("OpenClip supports str or PIL Image as query")
 
     def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = attempt_import_or_raise("torch")
+        torch = self.safe_import("torch")
         text = self.sanitize_input(text)
         text = self._tokenizer(text)
         text.to(self.device)
@@ -176,7 +175,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             The image to embed. If the image is a str, it is treated as a uri.
             If the image is bytes, it is treated as the raw image bytes.
         """
-        torch = attempt_import_or_raise("torch")
+        torch = self.safe_import("torch")
         # TODO handle retry and errors for https
         image = self._to_pil(image)
         image = self._preprocess(image).unsqueeze(0)
@@ -184,7 +183,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             return self._encode_and_normalize_image(image)
 
     def _to_pil(self, image: Union[str, bytes]):
-        PIL = attempt_import_or_raise("PIL", "pillow")
+        PIL = self.safe_import("PIL", "pillow")
         if isinstance(image, bytes):
             return PIL.Image.open(io.BytesIO(image))
         if isinstance(image, PIL.Image.Image):

docs/src/embeddings/default_embedding_functions.md

@@ -9,9 +9,6 @@ Contains the text embedding functions registered by default.
 ### Sentence transformers
 Allows you to set parameters when registering a `sentence-transformers` object.
 
-!!! info
-    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
-
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
@@ -19,156 +16,19 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
 
 
-??? "Check out available sentence-transformer models here!"
-    ```markdown
-    - sentence-transformers/all-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-mpnet-base-v2
-    - sentence-transformers/gtr-t5-base
-    - sentence-transformers/LaBSE
-    - sentence-transformers/all-MiniLM-L6-v2
-    - sentence-transformers/bert-base-nli-max-tokens
-    - sentence-transformers/bert-base-nli-mean-tokens
-    - sentence-transformers/bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/bert-base-wikipedia-sections-mean-tokens
-    - sentence-transformers/bert-large-nli-cls-token
-    - sentence-transformers/bert-large-nli-max-tokens
-    - sentence-transformers/bert-large-nli-mean-tokens
-    - sentence-transformers/bert-large-nli-stsb-mean-tokens
-    - sentence-transformers/distilbert-base-nli-max-tokens
-    - sentence-transformers/distilbert-base-nli-mean-tokens
-    - sentence-transformers/distilbert-base-nli-stsb-mean-tokens
-    - sentence-transformers/distilroberta-base-msmarco-v1
-    - sentence-transformers/distilroberta-base-msmarco-v2
-    - sentence-transformers/nli-bert-base-cls-pooling
-    - sentence-transformers/nli-bert-base-max-pooling
-    - sentence-transformers/nli-bert-base
-    - sentence-transformers/nli-bert-large-cls-pooling
-    - sentence-transformers/nli-bert-large-max-pooling
-    - sentence-transformers/nli-bert-large
-    - sentence-transformers/nli-distilbert-base-max-pooling
-    - sentence-transformers/nli-distilbert-base
-    - sentence-transformers/nli-roberta-base
-    - sentence-transformers/nli-roberta-large
-    - sentence-transformers/roberta-base-nli-mean-tokens
-    - sentence-transformers/roberta-base-nli-stsb-mean-tokens
-    - sentence-transformers/roberta-large-nli-mean-tokens
-    - sentence-transformers/roberta-large-nli-stsb-mean-tokens
-    - sentence-transformers/stsb-bert-base
-    - sentence-transformers/stsb-bert-large
-    - sentence-transformers/stsb-distilbert-base
-    - sentence-transformers/stsb-roberta-base
-    - sentence-transformers/stsb-roberta-large
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-base-en-ko-nli-ststb
-    - sentence-transformers/xlm-r-bert-base-nli-mean-tokens
-    - sentence-transformers/xlm-r-bert-base-nli-stsb-mean-tokens
-    - sentence-transformers/xlm-r-large-en-ko-nli-ststb
-    - sentence-transformers/bert-base-nli-cls-token
-    - sentence-transformers/all-distilroberta-v1
-    - sentence-transformers/multi-qa-MiniLM-L6-dot-v1
-    - sentence-transformers/multi-qa-distilbert-cos-v1
-    - sentence-transformers/multi-qa-distilbert-dot-v1
-    - sentence-transformers/multi-qa-mpnet-base-cos-v1
-    - sentence-transformers/multi-qa-mpnet-base-dot-v1
-    - sentence-transformers/nli-distilroberta-base-v2
-    - sentence-transformers/all-MiniLM-L6-v1
-    - sentence-transformers/all-mpnet-base-v1
-    - sentence-transformers/all-mpnet-base-v2
-    - sentence-transformers/all-roberta-large-v1
-    - sentence-transformers/allenai-specter
-    - sentence-transformers/average_word_embeddings_glove.6B.300d
-    - sentence-transformers/average_word_embeddings_glove.840B.300d
-    - sentence-transformers/average_word_embeddings_komninos
-    - sentence-transformers/average_word_embeddings_levy_dependency
-    - sentence-transformers/clip-ViT-B-32-multilingual-v1
-    - sentence-transformers/clip-ViT-B-32
-    - sentence-transformers/distilbert-base-nli-stsb-quora-ranking
-    - sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking
-    - sentence-transformers/distilroberta-base-paraphrase-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v1
-    - sentence-transformers/distiluse-base-multilingual-cased-v2
-    - sentence-transformers/distiluse-base-multilingual-cased
-    - sentence-transformers/facebook-dpr-ctx_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base
-    - sentence-transformers/facebook-dpr-question_encoder-multiset-base
-    - sentence-transformers/facebook-dpr-question_encoder-single-nq-base
-    - sentence-transformers/gtr-t5-large
-    - sentence-transformers/gtr-t5-xl
-    - sentence-transformers/gtr-t5-xxl
-    - sentence-transformers/msmarco-MiniLM-L-12-v3
-    - sentence-transformers/msmarco-MiniLM-L-6-v3
-    - sentence-transformers/msmarco-MiniLM-L12-cos-v5
-    - sentence-transformers/msmarco-MiniLM-L6-cos-v5
-    - sentence-transformers/msmarco-bert-base-dot-v5
-    - sentence-transformers/msmarco-bert-co-condensor
-    - sentence-transformers/msmarco-distilbert-base-dot-prod-v3
-    - sentence-transformers/msmarco-distilbert-base-tas-b
-    - sentence-transformers/msmarco-distilbert-base-v2
-    - sentence-transformers/msmarco-distilbert-base-v3
-    - sentence-transformers/msmarco-distilbert-base-v4
-    - sentence-transformers/msmarco-distilbert-cos-v5
-    - sentence-transformers/msmarco-distilbert-dot-v5
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-lng-aligned
-    - sentence-transformers/msmarco-distilbert-multilingual-en-de-v2-tmp-trained-scratch
-    - sentence-transformers/msmarco-distilroberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-ance-firstp
-    - sentence-transformers/msmarco-roberta-base-v2
-    - sentence-transformers/msmarco-roberta-base-v3
-    - sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    - sentence-transformers/nli-mpnet-base-v2
-    - sentence-transformers/nli-roberta-base-v2
-    - sentence-transformers/nq-distilbert-base-v1
-    - sentence-transformers/paraphrase-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-MiniLM-L3-v2
-    - sentence-transformers/paraphrase-MiniLM-L6-v2
-    - sentence-transformers/paraphrase-TinyBERT-L6-v2
-    - sentence-transformers/paraphrase-albert-base-v2
-    - sentence-transformers/paraphrase-albert-small-v2
-    - sentence-transformers/paraphrase-distilroberta-base-v1
-    - sentence-transformers/paraphrase-distilroberta-base-v2
-    - sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
-    - sentence-transformers/paraphrase-multilingual-mpnet-base-v2
-    - sentence-transformers/paraphrase-xlm-r-multilingual-v1
-    - sentence-transformers/quora-distilbert-base
-    - sentence-transformers/quora-distilbert-multilingual
-    - sentence-transformers/sentence-t5-base
-    - sentence-transformers/sentence-t5-large
-    - sentence-transformers/sentence-t5-xxl
-    - sentence-transformers/sentence-t5-xl
-    - sentence-transformers/stsb-distilroberta-base-v2
-    - sentence-transformers/stsb-mpnet-base-v2
-    - sentence-transformers/stsb-roberta-base-v2
-    - sentence-transformers/stsb-xlm-r-multilingual
-    - sentence-transformers/xlm-r-distilroberta-base-paraphrase-v1
-    - sentence-transformers/clip-ViT-L-14
-    - sentence-transformers/clip-ViT-B-16
-    - sentence-transformers/use-cmlm-multilingual
-    - sentence-transformers/all-MiniLM-L12-v1
-    ```
-
-!!! info
-    You can also load many other model architectures from the library. For example models from sources such as BAAI, nomic, salesforce research, etc.
-    See this HF hub page for all [supported models](https://huggingface.co/models?library=sentence-transformers).
-
-!!! note "BAAI Embeddings example"
-    Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
 ```python
-    import lancedb
-    from lancedb.pydantic import LanceModel, Vector
-    from lancedb.embeddings import get_registry
-
 db = lancedb.connect("/tmp/db")
-    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("sentence-transformers").create(device="cpu")
 
 class Words(LanceModel):
-        text: str = model.SourceField()
+    text: str = func.SourceField()
-        vector: Vector(model.ndims()) = model.VectorField()
+    vector: Vector(func.ndims()) = func.VectorField()
 
 table = db.create_table("words", schema=Words)
 table.add(
     [
-            {"text": "hello world"},
+        {"text": "hello world"}
         {"text": "goodbye world"}
     ]
 )
@@ -177,72 +37,6 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
-Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
-
-
-### Huggingface embedding models
-We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
-
-Example usage - 
-```python
-import lancedb
-import pandas as pd
-
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-model = get_registry().get("huggingface").create(name='facebook/bart-base')
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
-table = db.create_table("greets", schema=Words)
-table.add()
-query = "old greeting"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-
-### Ollama embeddings
-Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
-
-- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
-- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
-
-| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
-|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
-| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
-| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
-| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
-| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("ollama").create(name="nomic-embed-text")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add([
-    {"text": "hello world"},
-    {"text": "goodbye world"}
-])
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
 
 ### OpenAI embeddings
 LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
@@ -250,25 +44,21 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
-| `dim` | `int` |  Model default   | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
 
 
 ```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
 db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("openai").create()
 
 class Words(LanceModel):
     text: str = func.SourceField()
     vector: Vector(func.ndims()) = func.VectorField()
 
-table = db.create_table("words", schema=Words, mode="overwrite")
+table = db.create_table("words", schema=Words)
 table.add(
     [
-        {"text": "hello world"},
+        {"text": "hello world"}
         {"text": "goodbye world"}
     ]
     )
@@ -329,7 +119,7 @@ texts = [{"text": "Capitalism has been dominant in the Western world since the e
 tbl.add(texts)
 ```
 
-### Gemini Embeddings
+## Gemini Embedding Function
 With Google's Gemini, you can represent text (words, sentences, and blocks of text) in a vectorized form, making it easier to compare and contrast embeddings. For example, two texts that share a similar subject matter or sentiment should have similar embeddings, which can be identified through mathematical comparison techniques such as cosine similarity. For more on how and why you should use embeddings, refer to the Embeddings guide.
 The Gemini Embedding Model API supports various task types:
 
@@ -365,118 +155,6 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```
 
-### Cohere Embeddings
-Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
-You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
-
-Supported models are:
-```
-    * embed-english-v3.0
-    * embed-multilingual-v3.0
-    * embed-english-light-v3.0
-    * embed-multilingual-light-v3.0
-    * embed-english-v2.0
-    * embed-english-light-v2.0
-    * embed-multilingual-v2.0
-```
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
-| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
-| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
-
-Cohere supports following input types:
-| Input Type               | Description                          |
-|-------------------------|---------------------------------------|
-| "`search_document`"     | Used for embeddings stored in a vector|
-|                         | database for search use-cases.        |
-| "`search_query`"        | Used for embeddings of search queries |
-|                         | run against a vector DB               |
-| "`semantic_similarity`" | Specifies the given text will be used |
-|                         | for Semantic Textual Similarity (STS) |
-| "`classification`"      | Used for embeddings passed through a  |
-|                         | text classifier.                      |
-| "`clustering`"          | Used for the embeddings run through a |
-|                         | clustering algorithm                  |
-
-Usage Example:
-    
-    ```python
-        import lancedb
-        from lancedb.pydantic import LanceModel, Vector
-        from lancedb.embeddings import EmbeddingFunctionRegistry
-
-        cohere = EmbeddingFunctionRegistry
-            .get_instance()
-            .get("cohere")
-            .create(name="embed-multilingual-v2.0")
-
-        class TextModel(LanceModel):
-            text: str = cohere.SourceField()
-            vector: Vector(cohere.ndims()) =  cohere.VectorField()
-
-        data = [ { "text": "hello world" },
-                { "text": "goodbye world" }]
-
-        db = lancedb.connect("~/.lancedb")
-        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-        tbl.add(data)
-    ```
-
-### AWS Bedrock Text Embedding Functions
-AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
-You can do so by using `awscli` and also add your session_token:
-```shell
-aws configure
-aws configure set aws_session_token "<your_session_token>"
-```
-to ensure that the credentials are set up correctly, you can run the following command:
-```shell
-aws sts get-caller-identity
-```
-
-Supported Embedding modelIDs are:
-* `amazon.titan-embed-text-v1`
-* `cohere.embed-english-v3`
-* `cohere.embed-multilingual-v3`
-
-Supported parameters (to be passed in `create` method) are:
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| **name** | str | "amazon.titan-embed-text-v1" | The model ID of the bedrock model to use. Supported base models for Text Embeddings: amazon.titan-embed-text-v1, cohere.embed-english-v3, cohere.embed-multilingual-v3 |
-| **region** | str | "us-east-1" | Optional name of the AWS Region in which the service should be called (e.g., "us-east-1"). |
-| **profile_name** | str | None | Optional name of the AWS profile to use for calling the Bedrock service. If not specified, the default profile will be used. |
-| **assumed_role** | str | None | Optional ARN of an AWS IAM role to assume for calling the Bedrock service. If not specified, the current active credentials will be used. |
-| **role_session_name** | str | "lancedb-embeddings" | Optional name of the AWS IAM role session to use for calling the Bedrock service. If not specified, a "lancedb-embeddings" name will be used. |
-| **runtime** | bool | True | Optional choice of getting different client to perform operations with the Amazon Bedrock service. |
-| **max_retries** | int | 7 | Optional number of retries to perform when a request fails. |
-
-Usage Example:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-model = get_registry().get("bedrock-text").create()
-
-class TextModel(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
-db = lancedb.connect("tmp_path")
-tbl = db.create_table("test", schema=TextModel, mode="overwrite")
-
-tbl.add(df)
-rs = tbl.search("hello").limit(1).to_pandas()
-```
-
 ## Multi-modal embedding functions
 Multi-modal embedding functions allow you to query your table using both images and text.
 
@@ -496,13 +174,12 @@ This embedding function supports ingesting images as both bytes and urls. You ca
 !!! info
     LanceDB supports ingesting images directly from accessible links.
 
+
 ```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
 
 db = lancedb.connect(tmp_path)
-func = get_registry.get("open-clip").create()
+registry = EmbeddingFunctionRegistry.get_instance()
+func = registry.get("open-clip").create()
 
 class Images(LanceModel):
     label: str
@@ -563,70 +240,4 @@ print(actual.label)
 
 ```
 
-### Imagebind embeddings
-We have support for [imagebind](https://github.com/facebookresearch/ImageBind) model embeddings. You can download our version of the packaged model via - `pip install imagebind-packaged==0.1.2`.
-
-This function is registered as `imagebind` and supports Audio, Video and Text modalities(extending to Thermal,Depth,IMU data):
-
-| Parameter | Type | Default Value | Description |
-|---|---|---|---|
-| `name` | `str` | `"imagebind_huge"` | Name of the model. |
-| `device` | `str` | `"cpu"` | The device to run the model on. Can be `"cpu"` or `"gpu"`. |
-| `normalize` | `bool` | `False` | set to `True` to normalize your inputs before model ingestion. |
-
-Below is an example demonstrating how the API works:
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect(tmp_path)
-func = get_registry.get("imagebind").create()
-
-class ImageBindModel(LanceModel):
-    text: str
-    image_uri: str = func.SourceField()
-    audio_path: str
-    vector: Vector(func.ndims()) = func.VectorField()
-
-# add locally accessible image paths
-text_list=["A dog.", "A car", "A bird"]
-image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
-audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
-
-# Load data
-inputs = [
-    {"text": a, "audio_path": b, "image_uri": c}
-    for a, b, c in zip(text_list, audio_paths, image_paths)
-]
-
-#create table and add data
-table = db.create_table("img_bind", schema=ImageBindModel)
-table.add(inputs)
-```
-
-Now, we can search using any modality:
-
-#### image search
-```python
-query_image = "./assets/dog_image2.jpg" #download an image and enter that path here
-actual = table.search(query_image).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "dog")
-```
-#### audio search
-
-```python
-query_audio = "./assets/car_audio2.wav" #download an audio clip and enter path here
-actual = table.search(query_audio).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "car")
-```
-#### Text search
-You can add any input query and fetch the result as follows:
-```python
-query = "an animal which flies and tweets" 
-actual = table.search(query).limit(1).to_pydantic(ImageBindModel)[0]
-print(actual.text == "bird")
-```
-
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).

docs/src/embeddings/embedding_explicit.md

@@ -0,0 +1,141 @@
+In this workflow, you define your own embedding function and pass it as a callable to LanceDB, invoking it in your code to generate the embeddings. Let's look at some examples.
+
+### Hugging Face
+
+!!! note
+    Currently, the Hugging Face method is only supported in the Python SDK.
+
+=== "Python"
+    The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
+    library, which can be installed via pip.
+
+    ```bash
+    pip install sentence-transformers
+    ```
+
+    The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
+    for a given document.
+
+    ```python
+    from sentence_transformers import SentenceTransformer
+
+    name="paraphrase-albert-small-v2"
+    model = SentenceTransformer(name)
+
+    # used for both training and querying
+    def embed_func(batch):
+        return [model.encode(sentence) for sentence in batch]
+    ```
+
+### OpenAI
+
+Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
+
+=== "Python"
+      ```python
+        import openai
+        import os
+
+        # Configuring the environment variable OPENAI_API_KEY
+        if "OPENAI_API_KEY" not in os.environ:
+        # OR set the key here as a variable
+        openai.api_key = "sk-..."
+
+        # verify that the API key is working
+        assert len(openai.Model.list()["data"]) > 0
+
+        def embed_func(c):
+            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+            return [record["embedding"] for record in rs["data"]]
+      ```
+
+=== "JavaScript"
+      ```javascript
+        const lancedb = require("vectordb");
+
+        // You need to provide an OpenAI API key
+        const apiKey = "sk-..."
+        // The embedding function will create embeddings for the 'text' column
+        const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+      ```
+
+## Applying an embedding function to data
+
+=== "Python"
+    Using an embedding function, you can apply it to raw data
+    to generate embeddings for each record.
+
+    Say you have a pandas DataFrame with a `text` column that you want embedded,
+    you can use the `with_embeddings` function to generate embeddings and add them to 
+    an existing table.
+
+    ```python
+     import pandas as pd
+     from lancedb.embeddings import with_embeddings
+
+     df = pd.DataFrame(
+        [
+            {"text": "pepperoni"},
+            {"text": "pineapple"}
+        ]
+    )
+     data = with_embeddings(embed_func, df)
+
+     # The output is used to create / append to a table
+     # db.create_table("my_table", data=data)
+    ```
+
+    If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
+
+    By default, LanceDB calls the function with batches of 1000 rows. This can be configured
+    using the `batch_size` parameter to `with_embeddings`.
+
+    LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
+    API call is reliable.
+
+=== "JavaScript"
+    Using an embedding function, you can apply it to raw data
+    to generate embeddings for each record.
+
+    Simply pass the embedding function created above and LanceDB will use it to generate
+    embeddings for your data.
+
+    ```javascript
+    const db = await lancedb.connect("data/sample-lancedb");
+    const data = [
+    { text: "pepperoni"},
+    { text: "pineapple"}
+    ]
+
+    const table = await db.createTable("vectors", data, embedding)
+    ```
+
+## Querying using an embedding function
+
+!!! warning
+    At query time, you **must** use the same embedding function you used to vectorize your data.
+    If you use a different embedding function, the embeddings will not reside in the same vector
+    space and the results will be nonsensical.
+
+=== "Python"
+     ```python
+     query = "What's the best pizza topping?"
+     query_vector = embed_func([query])[0]
+     results = (
+        tbl.search(query_vector)
+        .limit(10)
+        .to_pandas()
+     )
+     ```
+
+     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
+
+=== "JavaScript"
+     ```javascript
+      const results = await table
+        .search("What's the best pizza topping?")
+        .limit(10)
+        .execute()
+     ```
+
+     The above snippet returns an array of records with the top 10 nearest neighbors to the query.

docs/src/embeddings/embedding_functions.md

@@ -2,131 +2,63 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
-!!! Note "LanceDB cloud doesn't support embedding functions yet"
-    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
-
 !!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    Using the implicit embeddings management approach means that you can forget about the manually passing around embedding
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    functions in your code, as long as you don't intend to change it at a later time. If your embedding function changes,
-    and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
+    you'll have to re-configure your table with the new embedding function and regenerate the embeddings.
-    table metadata and have LanceDB automatically take care of regenerating the embeddings.
-
 
 ## 1. Define the embedding function
-
+We have some pre-defined embedding functions in the global registry, with more coming soon. Here's let's an implementation of CLIP as example.
-=== "Python"
+```
-    In the LanceDB python SDK, we define a global embedding function registry with
+registry = EmbeddingFunctionRegistry.get_instance()
-    many different embedding models and even more coming soon. 
-    Here's let's an implementation of CLIP as example.
-
-    ```python
-    from lancedb.embeddings import get_registry
-
-    registry = get_registry()
 clip = registry.get("open-clip").create()
+
 ```
-
+You can also define your own embedding function by implementing the `EmbeddingFunction` abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
-    You can also define your own embedding function by implementing the `EmbeddingFunction` 
-    abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
-
-=== "JavaScript""
-    In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
-    embedding function is available.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    // You need to provide an OpenAI API key
-    const apiKey = "sk-..."
-    // The embedding function will create embeddings for the 'text' column
-    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-    ```
 
 ## 2. Define the data model or schema
-
-=== "Python"
 The embedding function defined above abstracts away all the details about the models and dimensions required to define the schema. You can simply set a field as **source** or **vector** column. Here's how:
 
 ```python
 class Pets(LanceModel):
-        vector: Vector(clip.ndims()) = clip.VectorField()
+    vector: Vector(clip.ndims) = clip.VectorField()
     image_uri: str = clip.SourceField()
 ```
 
 `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-=== "JavaScript"
+## 3. Create LanceDB table
+Now that we have chosen/defined our embedding function and the schema, we can create the table:
 
-    For the TypeScript SDK, a schema can be inferred from input data, or an explicit
-    Arrow schema can be provided.
-
-## 3. Create table and add data
-
-Now that we have chosen/defined our embedding function and the schema, 
-we can create the table and ingest data without needing to explicitly generate
-the embeddings at all:
-
-=== "Python"
 ```python
 db = lancedb.connect("~/lancedb")
 table = db.create_table("pets", schema=Pets)
 
+```
+
+That's it! We've provided all the information needed to embed the source and query inputs. We can now forget about the model and dimension details and start to build our VectorDB pipeline.
+
+## 4. Ingest lots of data and query your table
+Any new or incoming data can just be added and it'll be vectorized automatically.
+
+```python
 table.add([{"image_uri": u} for u in uris])
 ```
 
-=== "JavaScript"
-
-    ```javascript
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
-
-    const table = await db.createTable("vectors", data, embedding)
-    ```
-
-## 4. Querying your table
-Not only can you forget about the embeddings during ingestion, you also don't
-need to worry about it when you query the table:
-
-=== "Python"
-
 Our OpenCLIP query embedding function supports querying via both text and images:
 
 ```python
-    results = (
+result = table.search("dog")
-        table.search("dog")
-        .limit(10)
-        .to_pandas()
-    )
 ```
 
-    Or we can search using an image:
+Let's query an image:
 
 ```python
 p = Path("path/to/images/samoyed_100.jpg")
 query_image = Image.open(p)
-    results = (
 table.search(query_image)
-        .limit(10)
-        .to_pandas()
-    )
 ```
 
-    Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
-
-=== "JavaScript"
-
-    ```javascript
-    const results = await table
-    .search("What's the best pizza topping?")
-    .limit(10)
-    .execute()
-    ```    
-    
-    The above snippet returns an array of records with the top 10 nearest neighbors to the query.
-
 ---
 
 ## Rate limit Handling
@@ -152,7 +84,7 @@ You can also use the integration for adding utility operations in the schema. Fo
 
 ```python
 class Pets(LanceModel):
-    vector: Vector(clip.ndims()) = clip.VectorField()
+    vector: Vector(clip.ndims) = clip.VectorField()
     image_uri: str = clip.SourceField()
 
     @property
@@ -168,5 +100,4 @@ rs[2].image
 
 ![](../assets/dog_clip_output.png)
 
-Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
+Now that you have the basic idea about implicit management via embedding functions, let's dive deeper into a [custom API](./api.md) that you can use to implement your own embedding functions.
-let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,74 +1,8 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. This makes them a very powerful tool for machine learning practitioners. However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
-This makes them a very powerful tool for machine learning practitioners. 
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
-(both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
-LanceDB supports 3 methods of working with embeddings.
+LanceDB supports 2 methods of vectorizing your raw data into embeddings.
 
-1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
+1. **Explicit**: By manually calling LanceDB's `with_embedding` function to vectorize your data via an `embed_func` of your choice
-2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
+2. **Implicit**: Allow LanceDB to embed the data and queries in the background as they come in, by using the table's `EmbeddingRegistry` information
-3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
-   that extends the default embedding functions.
 
-For python users, there is also a legacy [with_embeddings API](./legacy.md).
+See the [explicit](embedding_explicit.md) and [implicit](embedding_functions.md) embedding sections for more details.
-It is retained for compatibility and will be removed in a future version.
-
-## Quickstart
-
-To get started with embeddings, you can use the built-in embedding functions.
-
-### OpenAI Embedding function
-LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
-
-class Words(LanceModel):
-    text: str = func.SourceField()
-    vector: Vector(func.ndims()) = func.VectorField()
-
-table = db.create_table("words", schema=Words, mode="overwrite")
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-    )
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-)
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/embeddings/legacy.md

@@ -1,99 +0,0 @@
-The legacy `with_embeddings` API is for Python only and is deprecated.
-
-### Hugging Face
-
-The most popular open source option is to use the [sentence-transformers](https://www.sbert.net/) 
-library, which can be installed via pip.
-
-```bash
-pip install sentence-transformers
-```
-
-The example below shows how to use the `paraphrase-albert-small-v2` model to generate embeddings 
-for a given document.
-
-```python
-from sentence_transformers import SentenceTransformer
-
-name="paraphrase-albert-small-v2"
-model = SentenceTransformer(name)
-
-# used for both training and querying
-def embed_func(batch):
-    return [model.encode(sentence) for sentence in batch]
-```
-
-
-### OpenAI
-
-Another popular alternative is to use an external API like OpenAI's [embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings).
-
-```python
-import openai
-import os
-
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-openai.api_key = "sk-..."
-
-client = openai.OpenAI()
-
-def embed_func(c):    
-    rs = client.embeddings.create(input=c, model="text-embedding-ada-002")
-    return [record.embedding for record in rs["data"]]
-```
-
-
-## Applying an embedding function to data
-
-Using an embedding function, you can apply it to raw data
-to generate embeddings for each record.
-
-Say you have a pandas DataFrame with a `text` column that you want embedded,
-you can use the `with_embeddings` function to generate embeddings and add them to 
-an existing table.
-
-```python
-    import pandas as pd
-    from lancedb.embeddings import with_embeddings
-
-    df = pd.DataFrame(
-        [
-            {"text": "pepperoni"},
-            {"text": "pineapple"}
-        ]
-    )
-    data = with_embeddings(embed_func, df)
-
-    # The output is used to create / append to a table
-    tbl = db.create_table("my_table", data=data)
-```
-
-If your data is in a different column, you can specify the `column` kwarg to `with_embeddings`.
-
-By default, LanceDB calls the function with batches of 1000 rows. This can be configured
-using the `batch_size` parameter to `with_embeddings`.
-
-LanceDB automatically wraps the function with retry and rate-limit logic to ensure the OpenAI
-API call is reliable.
-
-## Querying using an embedding function
-
-!!! warning
-    At query time, you **must** use the same embedding function you used to vectorize your data.
-    If you use a different embedding function, the embeddings will not reside in the same vector
-    space and the results will be nonsensical.
-
-=== "Python"
-     ```python
-     query = "What's the best pizza topping?"
-     query_vector = embed_func([query])[0]
-     results = (
-        tbl.search(query_vector)
-        .limit(10)
-        .to_pandas()
-     )
-     ```
-
-     The above snippet returns a pandas DataFrame with the 10 closest vectors to the query.

docs/src/examples/examples_rust.md

@@ -1,3 +0,0 @@
-# Examples: Rust
-
-Our Rust SDK is now stable. Examples are coming soon.

docs/src/examples/image_embeddings_roboflow.md

@@ -43,7 +43,7 @@ pip install lancedb
 We also need to install a specific commit of `tantivy`, a dependency of the LanceDB full text search engine we will use later in this guide:
 
 ```
-pip install tantivy
+pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
 ```
 
 Create a new Python file and add the following code:

docs/src/examples/index.md

@@ -2,11 +2,10 @@
 
 ## Recipes and example code
 
-LanceDB provides language APIs, allowing you to embed a database in your language of choice.
+LanceDB provides language APIs, allowing you to embed a database in your language of choice. We currently provide Python and Javascript APIs, with the Rust API and examples actively being worked on and will be available soon.
 
 * 🐍 [Python](examples_python.md) examples
-* 👾 [JavaScript](examples_js.md) examples
+* 👾 [JavaScript](exampled_js.md) examples
-* 🦀 Rust examples (coming soon)
 
 ## Applications powered by LanceDB
 

docs/src/examples/modal_langchain.py

@@ -1,5 +1,6 @@
 import pickle
 import re
+import sys
 import zipfile
 from pathlib import Path
 
@@ -78,10 +79,7 @@ def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 

docs/src/extra_js/init_ask_ai_widget.js

@@ -1,79 +0,0 @@
-// Creates an SVG robot icon (from Lucide)
-function robotSVG() {
-  var svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
-  svg.setAttribute("width", "24");
-  svg.setAttribute("height", "24");
-  svg.setAttribute("viewBox", "0 0 24 24");
-  svg.setAttribute("fill", "none");
-  svg.setAttribute("stroke", "currentColor");
-  svg.setAttribute("stroke-width", "2");
-  svg.setAttribute("stroke-linecap", "round");
-  svg.setAttribute("stroke-linejoin", "round");
-  svg.setAttribute("class", "lucide lucide-bot-message-square");
-
-  var path1 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path1.setAttribute("d", "M12 6V2H8");
-  svg.appendChild(path1);
-
-  var path2 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path2.setAttribute("d", "m8 18-4 4V8a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2Z");
-  svg.appendChild(path2);
-
-  var path3 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path3.setAttribute("d", "M2 12h2");
-  svg.appendChild(path3);
-
-  var path4 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path4.setAttribute("d", "M9 11v2");
-  svg.appendChild(path4);
-
-  var path5 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path5.setAttribute("d", "M15 11v2");
-  svg.appendChild(path5);
-
-  var path6 = document.createElementNS("http://www.w3.org/2000/svg", "path");
-  path6.setAttribute("d", "M20 12h2");
-  svg.appendChild(path6);
-
-  return svg
-}
-
-// Creates the Fluidic Chatbot buttom
-function fluidicButton() {
-  var btn = document.createElement("a");
-  btn.href = "https://asklancedb.com";
-  btn.target = "_blank";
-  btn.style.position = "fixed";
-  btn.style.fontWeight = "bold";
-  btn.style.fontSize = ".8rem";
-  btn.style.right = "10px";
-  btn.style.bottom = "10px";
-  btn.style.width = "80px";
-  btn.style.height = "80px";
-  btn.style.background = "linear-gradient(135deg, #7C5EFF 0%, #625eff 100%)";
-  btn.style.color = "white";
-  btn.style.borderRadius = "5px";
-  btn.style.display = "flex";
-  btn.style.flexDirection = "column";
-  btn.style.justifyContent = "center";
-  btn.style.alignItems = "center";
-  btn.style.zIndex = "1000";
-  btn.style.opacity = "0";
-  btn.style.boxShadow = "0 0 0 rgba(0, 0, 0, 0)";
-  btn.style.transition = "opacity 0.2s ease-in, box-shadow 0.2s ease-in";
-
-  setTimeout(function() {
-      btn.style.opacity = "1";
-      btn.style.boxShadow = "0 0 .2rem #0000001a,0 .2rem .4rem #0003"
-  }, 0);
-
-  return btn
-}
-
-document.addEventListener("DOMContentLoaded", function() {
-  var btn = fluidicButton()
-  btn.appendChild(robotSVG());
-  var text = document.createTextNode("Ask AI");
-  btn.appendChild(text);
-  document.body.appendChild(btn);
-});

docs/src/faq.md

@@ -16,7 +16,7 @@ As we mention in our talk titled “[Lance, a modern columnar data format](https
 
 ### Why build in Rust? 🦀
 
-We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python, JavaScript, and Rust client libraries to interact with the database.
+We believe that the Rust ecosystem has attained mainstream maturity and that Rust will form the underpinnings of large parts of the data and ML landscape in a few years. Performance, latency and reliability are paramount to a vector DB, and building in Rust allows us to iterate and release updates more rapidly due to Rust’s safety guarantees. Both Lance (the data format) and LanceDB (the database) are written entirely in Rust. We also provide Python and JavaScript client libraries to interact with the database. Our Rust API is a little rough around the edges right now, but is fast becoming on par with the Python and JS APIs.
 
 ### What is the difference between LanceDB OSS and LanceDB Cloud?
 
@@ -40,11 +40,11 @@ LanceDB and its underlying data format, Lance, are built to scale to really larg
 
 No. LanceDB is blazing fast (due to its disk-based index) for even brute force kNN search, within reason. In our benchmarks, computing 100K pairs of 1000-dimension vectors takes less than 20ms. For small datasets of ~100K records or applications that can accept ~100ms latency, an ANN index is usually not necessary.
 
-For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index. See the [ANN indexes](ann_indexes.md) section for more details.
+For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
 
 ### Does LanceDB support full-text search?
 
-Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.  Follow along in the [Github issue](https://github.com/lancedb/lance/issues/1195)
+Yes, LanceDB supports full-text search (FTS) via [Tantivy](https://github.com/quickwit-oss/tantivy). Our current FTS integration is Python-only, and our goal is to push it down to the Rust level in future versions to enable much more powerful search capabilities available to our Python, JavaScript and Rust clients.
 
 ### How can I speed up data inserts?
 
@@ -69,19 +69,3 @@ MinIO supports an S3 compatible API. In order to connect to a MinIO instance, yo
 - Set the envvar `AWS_ENDPOINT` to the URL of your MinIO API
 - Set the envvars `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your MinIO credential
 - Call `lancedb.connect("s3://minio_bucket_name")`
-
-### Where can I find benchmarks for LanceDB?
-
-Refer to this [post](https://blog.lancedb.com/benchmarking-lancedb-92b01032874a) for recent benchmarks.
-
-### How much data can LanceDB practically manage without effecting performance?
-
-We target good performance on ~10-50 billion rows and ~10-30 TB of data.
-
-### Does LanceDB support concurrent operations?
-
-LanceDB can handle concurrent reads very well, and can scale horizontally. The main constraint is how well the [storage layer](https://lancedb.github.io/lancedb/concepts/storage/) you've chosen scales. For writes, we support concurrent writing, though too many concurrent writers can lead to failing writes as there is a limited number of times a writer retries a commit
-
-!!! info "Multiprocessing with LanceDB"
-
-    For multiprocessing you should probably not use ```fork``` as lance is multi-threaded internally and ```fork``` and multi-thread do not work well.[Refer to this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)

docs/src/fts.md

@@ -1,7 +1,8 @@
 # Full-text search
 
-LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
+LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for JavaScript users as well.
 
+A hybrid search solution combining vector and full-text search is also on the way.
 
 ## Installation
 
@@ -54,16 +55,6 @@ This returns the result as a list of dictionaries as follows.
 !!! note
     LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
-## Tokenization
-By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
-
-```python
-table.create_fts_index("text", tokenizer_name="en_stem")
-```
-
-The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
-
-
 ## Index multiple columns
 
 If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
@@ -84,70 +75,20 @@ applied on top of the full text search results. This can be invoked via the fami
 table.search("puppy").limit(10).where("meta='foo'").to_list()
 ```
 
-## Sorting
+## Syntax
 
-You can pre-sort the documents by specifying `ordering_field_names` when
+For full-text search you can perform either a phrase query like "the old man and the sea", 
-creating the full-text search index. Once pre-sorted, you can then specify
+or a structured search query like "(Old AND Man) AND Sea".
-`ordering_field_name` while searching to return results sorted by the given
+Double quotes are used to disambiguate.
-field. For example, 
 
-```
+For example: 
-table.create_fts_index(["text_field"], ordering_field_names=["sort_by_field"])
 
-(table.search("terms", ordering_field_name="sort_by_field")
+If you intended "they could have been dogs OR cats" as a phrase query, this actually
- .limit(20)
+raises a syntax error since `OR` is a recognized operator. If you make `or` lower case,
- .to_list())
+this avoids the syntax error. However, it is cumbersome to have to remember what will
-```
+conflict with the query syntax. Instead, if you search using 
-
+`table.search('"they could have been dogs OR cats"')`, then the syntax checker avoids
-!!! note
+checking inside the quotes.
-    If you wish to specify an ordering field at query time, you must also
-    have specified it during indexing time. Otherwise at query time, an
-    error will be raised that looks like `ValueError: The field does not exist: xxx`
-
-!!! note
-    The fields to sort on must be of typed unsigned integer, or else you will see 
-    an error during indexing that looks like 
-    `TypeError: argument 'value': 'float' object cannot be interpreted as an integer`.
-
-!!! note
-    You can specify multiple fields for ordering at indexing time.
-    But at query time only one ordering field is supported.
-
-
-## Phrase queries vs. terms queries
-
-For full-text search you can specify either a **phrase** query like `"the old man and the sea"`,
-or a **terms** search query like `"(Old AND Man) AND Sea"`. For more details on the terms
-query syntax, see Tantivy's [query parser rules](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
-
-!!! tip "Note"
-    The query parser will raise an exception on queries that are ambiguous. For example, in the query `they could have been dogs OR cats`, `OR` is capitalized so it's considered a keyword query operator. But it's ambiguous how the left part should be treated. So if you submit this search query as is, you'll get `Syntax Error: they could have been dogs OR cats`.
-
-    ```py
-    # This raises a syntax error
-    table.search("they could have been dogs OR cats")
-    ```
-
-    On the other hand, lowercasing `OR` to `or` will work, because there are no capitalized logical operators and
-    the query is treated as a phrase query.
-
-    ```py
-    # This works!
-    table.search("they could have been dogs or cats")
-    ```
-
-It can be cumbersome to have to remember what will cause a syntax error depending on the type of
-query you want to perform. To make this simpler, when you want to perform a phrase query, you can
-enforce it in one of two ways:
-
-1. Place the double-quoted query inside single quotes. For example, `table.search('"they could have been dogs OR cats"')` is treated as
-a phrase query.
-2. Explicitly declare the `phrase_query()` method. This is useful when you have a phrase query that
-itself contains double quotes. For example, `table.search('the cats OR dogs were not really "pets" at all').phrase_query()`
-is treated as a phrase query.
-
-In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
-double quotes replaced by single quotes.
 
 
 ## Configurations
@@ -171,3 +112,4 @@ table.create_fts_index(["text1", "text2"], writer_heap_size=heap, replace=True)
 2. We currently only support local filesystem paths for the FTS index. 
    This is a tantivy limitation. We've implemented an object store plugin
    but there's no way in tantivy-py to specify to use it.
+

docs/src/guides/storage.md

@@ -55,394 +55,37 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
     const db = await lancedb.connect("az://bucket/path");
     ```
 
-In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
+In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided using environment variables. In general, these environment variables are the same as those used by the respective cloud SDKs. The sections below describe the environment variables that can be used to configure each object store.
 
-```bash
+LanceDB OSS uses the [object-store](https://docs.rs/object_store/latest/object_store/) Rust crate for object store access. There are general environment variables that can be used to configure the object store, such as the request timeout and proxy configuration. See the [object_store ClientConfigKey](https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html) doc for available configuration options. The environment variables that can be set are the snake-cased versions of these variable names. For example, to set `ProxyUrl` use the environment variable `PROXY_URL`. (Don't let the Rust docs intimidate you! We link to them so you can see an up-to-date list of the available options.)
-export TIMEOUT=60s
-```
-
-!!! note "`storage_options` availability"
-
-    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
-    It is not yet supported in the Python synchronous API.
-
-If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={"timeout": "60s"}
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path",
-                                     {storageOptions: {timeout: "60s"}});
-    ```
-
-Getting even more specific, you can set the `timeout` for only a particular table:
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async("s3://bucket/path")
-    table = await db.create_table(
-        "table",
-        [{"a": 1, "b": 2}],
-        storage_options={"timeout": "60s"}
-    )
-    ```
-
-=== "JavaScript"
-
-    <!-- skip-test -->
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path");
-    const table = db.createTable(
-        "table",
-        [{ a: 1, b: 2}],
-        {storageOptions: {timeout: "60s"}}
-    );
-    ```
-
-!!! info "Storage option casing"
-
-    The storage option keys are case-insensitive. So `connect_timeout` and `CONNECT_TIMEOUT` are the same setting. Usually lowercase is used in the `storage_options` argument and uppercase is used for environment variables. In the `lancedb` Node package, the keys can also be provided in `camelCase` capitalization. For example, `connectTimeout` is equivalent to `connect_timeout`.
-
-### General configuration
-
-There are several options that can be set for all object stores, mostly related to network client configuration.
-
-<!-- from here: https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html -->
-
-| Key                        | Description                                                                                      |
-|----------------------------|--------------------------------------------------------------------------------------------------|
-| `allow_http`               | Allow non-TLS, i.e. non-HTTPS connections. Default: `False`.                                      |
-| `allow_invalid_certificates`| Skip certificate validation on HTTPS connections. Default: `False`.                               |
-| `connect_timeout`          | Timeout for only the connect phase of a Client. Default: `5s`.                                    |
-| `timeout`                  | Timeout for the entire request, from connection until the response body has finished. Default: `30s`. |
-| `user_agent`               | User agent string to use in requests.                                                             |
-| `proxy_url`                | URL of a proxy server to use for requests. Default: `None`.                                       |
-| `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
-| `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
 
 
 ### AWS S3
 
-To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
+To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables.
-These can be set as environment variables or passed in the `storage_options` parameter:
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "aws_access_key_id": "my-access-key",
-            "aws_secret_access_key": "my-secret-key",
-            "aws_session_token": "my-session-token",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://bucket/path",
-        {
-            storageOptions: {
-                awsAccessKeyId: "my-access-key",
-                awsSecretAccessKey: "my-secret-key",
-                awsSessionToken: "my-session-token",
-            }
-        }
-    );
-    ```
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
+You can see a full list of environment variables [here](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html#method.from_env).
-
-| Key                                | Description                                                                                          |
-|------------------------------------|------------------------------------------------------------------------------------------------------|
-| `aws_region` / `region`             | The AWS region the bucket is in. This can be automatically detected when using AWS S3, but must be specified for S3-compatible stores. |
-| `aws_access_key_id` / `access_key_id` | The AWS access key ID to use.                                                                       |
-| `aws_secret_access_key` / `secret_access_key` | The AWS secret access key to use.                                                               |
-| `aws_session_token` / `session_token` | The AWS session token to use.                                                                     |
-| `aws_endpoint` / `endpoint`         | The endpoint to use for S3-compatible stores.                                                       |
-| `aws_virtual_hosted_style_request` / `virtual_hosted_style_request` | Whether to use virtual hosted-style requests, where the bucket name is part of the endpoint. Meant to be used with `aws_endpoint`. Default: `False`. |
-| `aws_s3_express` / `s3_express`     | Whether to use S3 Express One Zone endpoints. Default: `False`. See more details below.             |
-| `aws_server_side_encryption`        | The server-side encryption algorithm to use. Must be one of `"AES256"`, `"aws:kms"`, or `"aws:kms:dsse"`. Default: `None`. |
-| `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
-| `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
-
-
-!!! tip "Automatic cleanup for failed writes"
-
-    LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
-
-    **[Configuring a bucket lifecycle configuration to delete incomplete multipart uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-abort-incomplete-mpu-lifecycle-config.html)**
-
-#### AWS IAM Permissions
-
-If a bucket is private, then an IAM policy must be specified to allow access to it. For many development scenarios, using broad permissions such as a PowerUser account is more than sufficient for working with LanceDB. However, in many production scenarios, you may wish to have as narrow as possible permissions.
-
-For **read and write access**, LanceDB will need a policy such as:
-
-```json
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-              "s3:PutObject",
-              "s3:GetObject",
-              "s3:DeleteObject",
-            ],
-            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
-        },
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:ListBucket",
-                "s3:GetBucketLocation"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>",
-            "Condition": {
-                "StringLike": {
-                    "s3:prefix": [
-                        "<prefix>/*"
-                    ]
-                }
-            }
-        }
-    ]
-}
-```
-
-For **read-only access**, LanceDB will need a policy such as:
-
-```json
-{
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": [
-              "s3:GetObject",
-            ],
-            "Resource": "arn:aws:s3:::<bucket>/<prefix>/*"
-        },
-        {
-            "Effect": "Allow",
-            "Action": [
-                "s3:ListBucket",
-                "s3:GetBucketLocation"
-            ],
-            "Resource": "arn:aws:s3:::<bucket>",
-            "Condition": {
-                "StringLike": {
-                    "s3:prefix": [
-                        "<prefix>/*"
-                    ]
-                }
-            }
-        }
-    ]
-}
-```
 
 #### S3-compatible stores
 
-LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
+LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify two environment variables: `AWS_ENDPOINT` and `AWS_DEFAULT_REGION`. `AWS_ENDPOINT` should be the URL of the S3-compatible store, and `AWS_DEFAULT_REGION` should be the region to use.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://bucket/path",
-        storage_options={
-            "region": "us-east-1",
-            "endpoint": "http://minio:9000",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://bucket/path",
-        {
-            storageOptions: {
-                region: "us-east-1",
-                endpoint: "http://minio:9000",
-            }
-        }
-    );
-    ```
-
-This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
-
-!!! tip "Local servers"
-
-    For local development, the server often has a `http` endpoint rather than a
-    secure `https` endpoint. In this case, you must also set the `ALLOW_HTTP`
-    environment variable to `true` to allow non-TLS connections, or pass the
-    storage option `allow_http` as `true`. If you do not do this, you will get
-    an error like `URL scheme is not allowed`.
-
-#### S3 Express
-
-LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
-
-To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
-
-=== "Python"
-
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "s3://my-bucket--use1-az4--x-s3/path",
-        storage_options={
-            "region": "us-east-1",
-            "s3_express": "true",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "s3://my-bucket--use1-az4--x-s3/path",
-        {
-            storageOptions: {
-                region: "us-east-1",
-                s3Express: "true",
-            }
-        }
-    );
-    ```
 
+<!-- TODO: we should also document the use of S3 Express once we fully support it -->
 
 ### Google Cloud Storage
 
-GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. Alternatively, you can pass the path to the JSON file in the `storage_options`:
+GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. There are several aliases for this environment variable, documented [here](https://docs.rs/object_store/latest/object_store/gcp/struct.GoogleCloudStorageBuilder.html#method.from_env).
-
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "gs://my-bucket/my-database",
-        storage_options={
-            "service_account": "path/to/service-account.json",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "gs://my-bucket/my-database",
-        {
-            storageOptions: {
-                serviceAccount: "path/to/service-account.json",
-            }
-        }
-    );
-    ```
 
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
-
-The following keys can be used as both environment variables or keys in the `storage_options` parameter:
-<!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
-
-| Key                                   | Description                                  |
-|---------------------------------------|----------------------------------------------|
-| ``google_service_account`` / `service_account` | Path to the service account JSON file.       |
-| ``google_service_account_key``        | The serialized service account key.          |
-| ``google_application_credentials``    | Path to the application credentials.         |
-
-
 ### Azure Blob Storage
 
-Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
+Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME` and ``AZURE_STORAGE_ACCOUNT_KEY`` environment variables. The full list of environment variables that can be set are documented [here](https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html#method.from_env).
 
-=== "Python"
-
-    <!-- skip-test -->
-    ```python
-    import lancedb
-    db = await lancedb.connect_async(
-        "az://my-container/my-database",
-        storage_options={
-            account_name: "some-account",
-            account_key: "some-key",
-        }
-    )
-    ```
-
-=== "JavaScript"
-
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect(
-        "az://my-container/my-database",
-        {
-            storageOptions: {
-                accountName: "some-account",
-                accountKey: "some-key",
-            }
-        }
-    );
-    ```
-
-These keys can be used as both environment variables or keys in the `storage_options` parameter:
-
-<!-- source: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html -->
-
-| Key                                   | Description                                                                                      |
-|---------------------------------------|--------------------------------------------------------------------------------------------------|
-| ``azure_storage_account_name``        | The name of the azure storage account.                                                           |
-| ``azure_storage_account_key``         | The serialized service account key.                                                              |
-| ``azure_client_id``                   | Service principal client id for authorizing requests.                                            |
-| ``azure_client_secret``               | Service principal client secret for authorizing requests.                                        |
-| ``azure_tenant_id``                   | Tenant id used in oauth flows.                                                                   |
-| ``azure_storage_sas_key``             | Shared access signature. The signature is expected to be percent-encoded, much like they are provided in the azure storage explorer or azure portal. |
-| ``azure_storage_token``               | Bearer token.                                                                                    |
-| ``azure_storage_use_emulator``        | Use object store with azurite storage emulator.                                                  |
-| ``azure_endpoint``                    | Override the endpoint used to communicate with blob storage.                                      |
-| ``azure_use_fabric_endpoint``         | Use object store with url scheme account.dfs.fabric.microsoft.com.                               |
-| ``azure_msi_endpoint``                | Endpoint to request a imds managed identity token.                                               |
-| ``azure_object_id``                   | Object id for use with managed identity authentication.                                          |
-| ``azure_msi_resource_id``             | Msi resource id for use with managed identity authentication.                                    |
-| ``azure_federated_token_file``        | File containing token for Azure AD workload identity federation.                                 |
-| ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
-| ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
 
 <!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables.md

@@ -16,23 +16,10 @@ This guide will show how to create tables, insert data into them, and update the
     db = lancedb.connect("./.lancedb")
     ```
 
-=== "Javascript"
-
-    Initialize a VectorDB connection and create a table using one of the many methods listed below.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
     ### From list of tuples or dictionaries
 
-=== "Python"
-
     ```python
     import lancedb
 
@@ -45,6 +32,7 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
+
     !!! info "Note"
         If the table already exists, LanceDB will raise an error by default. 
 
@@ -63,27 +51,6 @@ This guide will show how to create tables, insert data into them, and update the
         db.create_table("name", data, mode="overwrite")
         ```
 
-=== "Javascript"
-    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
-
-    ```javascript
-    const tb = await db.createTable("my_table", [{
-        "vector": [3.1, 4.1],
-        "item": "foo",
-        "price": 10.0
-    }, {
-        "vector": [5.9, 26.5],
-        "item": "bar",
-        "price": 20.0
-    }]);
-    ```
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
-    ```
-
     ### From a Pandas DataFrame
 
     ```python
@@ -102,8 +69,6 @@ This guide will show how to create tables, insert data into them, and update the
     !!! info "Note"
         Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
-    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
-
     ```python
     custom_schema = pa.schema([
     pa.field("vector", pa.list_(pa.float32(), 4)),
@@ -132,44 +97,26 @@ This guide will show how to create tables, insert data into them, and update the
     table = db.create_table("pl_table", data=data)
     ```
 
-### From an Arrow Table
+    ### From PyArrow Tables
-=== "Python"
+    You can also create LanceDB tables directly from PyArrow tables
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports float16 data type!
 
     ```python
-    import pyarrows as pa
+    table = pa.Table.from_arrays(
-    import numpy as np
-    
-    dim = 16
-    total = 2
-    schema = pa.schema(
             [
-            pa.field("vector", pa.list_(pa.float16(), dim)),
+                pa.array([[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
-            pa.field("text", pa.string())
+                        pa.list_(pa.float32(), 4)),
-        ]
+                pa.array(["foo", "bar"]),
-    )
+                pa.array([10.0, 20.0]),
-    data = pa.Table.from_arrays(
-        [
-            pa.array([np.random.randn(dim).astype(np.float16) for _ in range(total)],
-                    pa.list_(pa.float16(), dim)),
-            pa.array(["foo", "bar"])
             ],
-        ["vector", "text"],
+            ["vector", "item", "price"],
         )
-    tbl = db.create_table("f16_tbl", data, schema=schema)
-    ```
 
-=== "Javascript"
+    db = lancedb.connect("db")
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports Float16 data type!
 
-    ```javascript
+    tbl = db.create_table("my_table", table)
-    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
     ```
 
     ### From Pydantic Models
-
     When you create an empty table without data, you must specify the table schema.
     LanceDB supports creating tables by specifying a PyArrow schema or a specialized
     Pydantic model called `LanceModel`.
@@ -314,6 +261,37 @@ db.create_table("batched_tale", make_batches(), schema=schema)
 
     You can also use iterators of other types like Pandas DataFrame or Pylists directly in the above example.
 
+=== "JavaScript"
+    Initialize a VectorDB connection and create a table using one of the many methods listed below.
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const uri = "data/sample-lancedb";
+    const db = await lancedb.connect(uri);
+    ```
+
+    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
+
+    ```javascript
+    const tb = await db.createTable("my_table", [{
+        "vector": [3.1, 4.1],
+        "item": "foo",
+        "price": 10.0
+    }, {
+        "vector": [5.9, 26.5],
+        "item": "bar",
+        "price": 20.0
+    }]);
+    ```
+
+    !!! info "Note"
+    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
+
+    ```javascript
+    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```
+
 ## Open existing tables
 
 === "Python"
@@ -452,27 +430,6 @@ After a table has been created, you can always add more data to it using the var
     tbl.add(pydantic_model_items)
     ```
 
-    ??? "Ingesting Pydantic models with LanceDB embedding API"
-        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
-
-        ```python
-        import lancedb
-        from lancedb.pydantic import LanceModel, Vector
-        from lancedb.embeddings import get_registry
-
-        db = lancedb.connect("~/tmp")
-        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
-
-        class Schema(LanceModel):
-            text: str = embed_fcn.SourceField()
-            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
-
-        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
-        models = [Schema(text="hello"), Schema(text="world")]
-        tbl.add(models)
-        ```
-
-
 
 === "JavaScript"
 
@@ -657,95 +614,6 @@ The `values` parameter is used to provide the new values for the columns as lite
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
-## Drop a table
-
-Use the `drop_table()` method on the database to remove a table.
-
-=== "Python"
-
-      ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-      ```
-
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      By default, if the table does not exist an exception is raised. To suppress this,
-      you can pass in `ignore_missing=True`.
-
-=== "Javascript/Typescript"
-
-      ```typescript
-      --8<-- "docs/src/basic_legacy.ts:drop_table"
-      ```
-
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      If the table does not exist an exception is raised.
-
-
-## Consistency
-
-In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
-
-There are three possible settings for `read_consistency_interval`:
-
-1. **Unset (default)**: The database does not check for updates to tables made by other processes. This provides the best query performance, but means that clients may not see the most up-to-date data. This setting is suitable for applications where the data does not change during the lifetime of the table reference.
-2. **Zero seconds (Strong consistency)**: The database checks for updates on every read. This provides the strongest consistency guarantees, ensuring that all clients see the latest committed data. However, it has the most overhead. This setting is suitable when consistency matters more than having high QPS.
-3. **Custom interval (Eventual consistency)**: The database checks for updates at a custom interval, such as every 5 seconds. This provides eventual consistency, allowing for some lag between write and read operations. Performance wise, this is a middle ground between strong consistency and no consistency check. This setting is suitable for applications where immediate consistency is not critical, but clients should see updated data eventually.
-
-!!! tip "Consistency in LanceDB Cloud"
-
-    This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
-
-=== "Python"
-    
-    To set strong consistency, use `timedelta(0)`:
-
-    ```python
-    from datetime import timedelta
-    db = lancedb.connect("./.lancedb",. read_consistency_interval=timedelta(0))
-    table = db.open_table("my_table")
-    ```
-
-    For eventual consistency, use a custom `timedelta`:
-
-    ```python
-    from datetime import timedelta
-    db = lancedb.connect("./.lancedb", read_consistency_interval=timedelta(seconds=5))
-    table = db.open_table("my_table")
-    ```
-
-    By default, a `Table` will never check for updates from other writers. To manually check for updates you can use `checkout_latest`:
-
-    ```python
-    db = lancedb.connect("./.lancedb")
-    table = db.open_table("my_table")
-    
-    # (Other writes happen to my_table from another process)
-
-    # Check for updates
-    table.checkout_latest()
-    ```
-
-=== "JavaScript/Typescript"
-
-    To set strong consistency, use `0`:
-
-    ```javascript
-    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
-    const table = await db.openTable("my_table");
-    ```
-
-    For eventual consistency, specify the update interval as seconds:
-
-    ```javascript
-    const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
-    const table = await db.openTable("my_table");
-    ```
-
-<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
-    Once it does, we can show manual consistency check for Node as well.
--->
-
 ## What's next?
 
 Learn the best practices on creating an ANN index and getting the most out of it.

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,128 +0,0 @@
-## Improving retriever performance
-VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
-
-There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
-
-* Using different query types
-* Using hybrid search
-* Fine-tuning the embedding models
-* Using different embedding models
-
-Using different embedding models is something that's very specific to the use case and the data. So we will not discuss it here. In this section, we will discuss the first three techniques.
-
-
-!!! note "Note"
-    We'll be using a simple metric called "hit-rate" for evaluating the performance of the retriever across this guide. Hit-rate is the percentage of queries for which the retriever returned the correct answer in the top-k results. For example, if the retriever returned the correct answer in the top-3 results for 70% of the queries, then the hit-rate@3 is 0.7.
-
-
-## The dataset
-We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
-
-### Using different query types
-Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
-
-```python
-import lancedb
-import pandas as pd
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import Vector, LanceModel
-
-db = lancedb.connect("~/lancedb/query_types")
-df = pd.read_csv("data_qa.csv")
-
-embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.")
-
-class Schema(LanceModel):
-    context: str = embed_fcn.SourceField()
-    vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField()
-
-table = db.create_table("qa", schema=Schema)
-table.add(df[["context"]].to_dict(orient="records"))
-
-queries = df["query"].tolist()
-```
-
-Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
-
-* <b> Vector Search: </b>
-
-    ```python
-    table.search(quries[0], query_type="vector").limit(5).to_pandas()
-    ```
-    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
-
-    ```python
-    table.search(quries[0]).limit(5).to_pandas()
-    ```
-
-    Vector or semantic search is useful when you want to find documents that are similar to the query in terms of meaning.
-
----
-
-* <b> Full-text Search: </b>
-    
-    FTS requires creating an index on the column you want to search on. `replace=True` will replace the existing index if it exists.
-    Once the index is created, you can search using the `fts` query type.
-    ```python
-    table.create_fts_index("context", replace=True)
-    table.search(quries[0], query_type="fts").limit(5).to_pandas()
-    ```
-
-    Full-text search is useful when you want to find documents that contain the query terms.
-
----
-
-* <b> Hybrid Search: </b>
-
-    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
-    ```python
-    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
-    ```
-    Hybrid search requires a reranker to combine and rank the results from vector and full-text search. We'll cover reranking as a concept in the next section.
-
-    Hybrid search is useful when you want to combine the benefits of both vector and full-text search.
-
-    !!! note "Note"
-        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
-        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
-
-    
-
-### Hit rate evaluation results
-
-Now that we have seen how to run different query types on the dataset, let's evaluate the hit-rate of each query type on the dataset.
-For brevity, the entire evaluation script is not shown here. You can find the complete evaluation and benchmarking utility scripts [here](https://github.com/lancedb/ragged).
-
-Here are the hit-rate results for the dataset:
-
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector Search | 0.640 |
-| Full-text Search | 0.595 |
-| Hybrid Search (w/ LinearCombinationReranker) | 0.645 |
-
-**Choosing query type** is very specific to the use case and the data. This synthetic dataset has been generated to be semantically challenging, i.e, the queries don't have a lot of keywords in common with the context. So, vector search performs better than full-text search. However, in real-world scenarios, full-text search might perform better than vector search. Hybrid search is a good choice when you want to combine the benefits of both vector and full-text search.
-
-### Evaluation results on other datasets
-
-The hit-rate results can vary based on the dataset and the query type. Here are the hit-rate results for the other datasets using the same embedding function.
-
-* <b> SQuAD Dataset: </b>
-
-    | Query Type | Hit-rate@5 |
-    | --- | --- |
-    | Vector Search | 0.822 |
-    | Full-text Search | 0.835 |
-    | Hybrid Search (w/ LinearCombinationReranker) | 0.8874 |
-
-* <b> Uber10K sec filing Dataset: </b>
-
-    | Query Type | Hit-rate@5 |
-    | --- | --- |
-    | Vector Search | 0.608 |
-    | Full-text Search | 0.82 |
-    | Hybrid Search (w/ LinearCombinationReranker) | 0.80 |
-
-In these standard datasets, FTS seems to perform much better than vector search because the queries have a lot of keywords in common with the context. So, in general choosing the query type is very specific to the use case and the data.
-
-

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,78 +0,0 @@
-Continuing from the previous example, we can now rerank the results using more complex rerankers.
-
-## Reranking search results
-You can rerank any search results using a reranker. The syntax for reranking is as follows:
-
-```python
-from lancedb.rerankers import LinearCombinationReranker
-
-reranker = LinearCombinationReranker()
-table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
-```
-Based on the `query_type`, the `rerank()` function can accept other arguments as well. For example, hybrid search accepts a `normalize` param to determine the score normalization method.
-
-!!! note "Note"
-    LanceDB provides a `Reranker` base class that can be extended to implement custom rerankers. Each reranker must implement the `rerank_hybrid` method. `rerank_vector` and `rerank_fts` methods are optional. For example, the `LinearCombinationReranker` only implements the `rerank_hybrid` method and so it can only be used for reranking hybrid search results.
-
-## Choosing a Reranker
-There are many rerankers available in LanceDB like `CrossEncoderReranker`, `CohereReranker`, and `ColBERT`. The choice of reranker depends on the dataset and the application. You can even implement you own custom reranker by extending the `Reranker` class. For more details about each available reranker and performance comparison, refer to the [rerankers](https://lancedb.github.io/lancedb/reranking/) documentation.
-
-In this example, we'll use the `CohereReranker` to rerank the search results. It requires  `cohere` to be installed and `COHERE_API_KEY` to be set in the environment. To get your API key, sign up on [Cohere](https://cohere.ai/).
-
-```python
-from lancedb.rerankers import CohereReranker
-
-# use Cohere reranker v3
-reranker = CohereReranker(model_name="rerank-english-v3.0") # default model is "rerank-english-v2.0"
-```
-
-### Reranking search results
-Now we can rerank all query type results using the `CohereReranker`:
-
-```python
-
-# rerank hybrid search results
-table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
-
-# rerank vector search results
-table.search(quries[0], query_type="vector").rerank(reranker=reranker).limit(5).to_pandas()
-
-# rerank fts search results
-table.search(quries[0], query_type="fts").rerank(reranker=reranker).limit(5).to_pandas()
-```
-
-Each reranker can accept additional arguments. For example, `CohereReranker` accepts `top_k` and `batch_size` params to control the number of documents to rerank and the batch size for reranking respectively. Similarly, a custom reranker can accept any number of arguments based on the implementation. For example, a reranker can accept a `filter` that implements some custom logic to filter out documents before reranking.
-
-## Results
-
-Let us take a look at the same datasets from the previous sections, using the same embedding table but with Cohere reranker applied to all query types.
-
-!!! note "Note"
-    When reranking fts or vector search results, the search results are over-fetched by a factor of 2 and then reranked. From the reranked set, `top_k` (5 in this case) results are taken. This is done because reranking will have no effect on the hit-rate if we only fetch the `top_k` results.
-
-### Synthetic LLama2 paper dataset
-
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector |  0.640 |
-| FTS   |  0.595  |
-| Reranked vector | 0.677    |
-| Reranked fts  | 0.672    |
-| Hybrid | 0.759 |
-
-### SQuAD Dataset
-
-
-### Uber10K sec filing Dataset
-
-| Query Type | Hit-rate@5 |
-| --- | --- |
-| Vector |  0.608 |
-| FTS   |  0.824  |
-| Reranked vector | 0.671    |
-| Reranked fts  | 0.843    |
-| Hybrid | 0.849 |
-
-
-
-

docs/src/hybrid_search/eval.md

@@ -1,51 +0,0 @@
-# Hybrid Search
-
-Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
-
-## The challenge of (re)ranking search results
-Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
-There are two approaches for reranking search results from multiple sources.
-
-* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
-
-* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
-
-Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
-
-### Example evaluation of hybrid search with Reranking
-
-Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
-
-<b> With OpenAI ada2 embedding </b>
-
-Vector Search baseline - `0.64`
-
-| Reranker | Top-3 | Top-5 | Top-10 |
-| --- | --- | --- | --- |
-| Linear Combination | `0.73` | `0.74` | `0.85` |
-| Cross Encoder | `0.71` | `0.70` | `0.77` |
-| Cohere | `0.81` | `0.81` | `0.85` |
-| ColBERT | `0.68` | `0.68` | `0.73` |
-
-<p>
-<img src="https://github.com/AyushExel/assets/assets/15766192/d57b1780-ef27-414c-a5c3-73bee7808a45">
-</p>
-
-<b> With OpenAI embedding-v3-small </b>
-
-Vector Search baseline - `0.59`
-
-| Reranker | Top-3 | Top-5 | Top-10 |
-| --- | --- | --- | --- |
-| Linear Combination | `0.68` | `0.70` | `0.84` |
-| Cross Encoder | `0.72` | `0.72` | `0.79` |
-| Cohere | `0.79` | `0.79` | `0.84` |
-| ColBERT | `0.70` | `0.70` | `0.76` |
-
-<p>
-<img src="https://github.com/AyushExel/assets/assets/15766192/259adfd2-6ec6-4df6-a77d-1456598970dd">
-</p>
-
-### Conclusion
-
-The results show that the reranking methods are able to improve the search results. However, the improvement is not consistent across all rerankers. The choice of reranker depends on the dataset and the application. It is also important to note that the reranking methods are not a replacement for the search methods. They are complementary and should be used together to get the best results. The speed to recall tradeoff is also an important factor to consider when choosing the reranker.

docs/src/hybrid_search/hybrid_search.md

@@ -1,242 +0,0 @@
-# Hybrid Search
-
-LanceDB supports both semantic and keyword-based search (also termed full-text search, or FTS). In real world applications, it is often useful to combine these two approaches to get the best best results. For example, you may want to search for a document that is semantically similar to a query document, but also contains a specific keyword. This is an example of *hybrid search*, a search algorithm that combines multiple search techniques.
-
-## Hybrid search in LanceDB
-You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
-
-```python
-import os
-
-import lancedb
-import openai
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-db = lancedb.connect("~/.lancedb")
-
-# Ingest embedding function in LanceDB table
-# Configuring the environment variable OPENAI_API_KEY
-if "OPENAI_API_KEY" not in os.environ:
-# OR set the key here as a variable
-    openai.api_key = "sk-..."
-embeddings = get_registry().get("openai").create()
-
-class Documents(LanceModel):
-    vector: Vector(embeddings.ndims()) = embeddings.VectorField()
-    text: str = embeddings.SourceField()
-
-table = db.create_table("documents", schema=Documents)
-
-data = [
-    { "text": "rebel spaceships striking from a hidden base"},
-    { "text": "have won their first victory against the evil Galactic Empire"},
-    { "text": "during the battle rebel spies managed to steal secret plans"},
-    { "text": "to the Empire's ultimate weapon the Death Star"}
-]
-
-# ingest docs with auto-vectorization
-table.add(data)
-
-# Create a fts index before the hybrid search
-table.create_fts_index("text")
-# hybrid search with default re-ranker
-results = table.search("flower moon", query_type="hybrid").to_pandas()
-```
-
-By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and rerank the results of semantic and full-text search. You can customize the hyperparameters as needed or write your own custom reranker. Here's how you can use any of the available rerankers:
-
-
-### `rerank()` arguments
-* `normalize`: `str`, default `"score"`:
-    The method to normalize the scores. Can be "rank" or "score". If "rank", the scores are converted to ranks and then normalized. If "score", the scores are normalized directly.
-* `reranker`: `Reranker`, default `LinearCombinationReranker(weight=0.7)`.
-    The reranker to use. If not specified, the default reranker is used.
-
-
-## Available Rerankers
-LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
-
-### Linear Combination Reranker
-This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
-
-
-```python
-from lancedb.rerankers import LinearCombinationReranker
-
-reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
-
-results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `weight`: `float`, default `0.7`:
-    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
-* `fill`: `float`, default `1.0`:
-        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
-        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
-* `return_score` : str, default `"relevance"`
-        options are "relevance" or "all"
-        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
-
-### Cohere Reranker
-This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
-
-```python
-from lancedb.rerankers import CohereReranker
-
-reranker = CohereReranker()
-
-results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : str, default `"rerank-english-v2.0"`
-        The name of the cross encoder model to use. Available cohere models are:
-        - rerank-english-v2.0
-        - rerank-multilingual-v2.0
-* `column` : str, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `top_n` : str, default `None`
-        The number of results to return. If None, will return all results.
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-### Cross Encoder Reranker
-This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
-
-```python
-from lancedb.rerankers import CrossEncoderReranker
-
-reranker = CrossEncoderReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-
-### Arguments
-----------------
-* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
-        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
-* `column` : str, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `device` : str, default `None`
-        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-
-### ColBERT Reranker
-This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
-
-ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
-
-```python
-from lancedb.rerankers import ColbertReranker
-
-reranker = ColbertReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
-        The name of the cross encoder model to use.
-* `column` : `str`, default `"text"`
-        The name of the column to use as input to the cross encoder model.
-* `return_score` : `str`, default `"relevance"`
-        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
-
-!!! Note
-    Only returns `_relevance_score`. Does not support `return_score = "all"`.
-
-### OpenAI Reranker
-This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
-
-!!! Note
-    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
-
-!!! Tip
-    - You might run out of token limit so set the search `limits` based on your token limit.
-    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
-
-```python
-from lancedb.rerankers import OpenaiReranker
-
-reranker = OpenaiReranker()
-
-results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
-```
-
-### Arguments
-----------------
-* `model_name` : `str`, default `"gpt-4-turbo-preview"`
-    The name of the cross encoder model to use.
-* `column` : `str`, default `"text"`
-    The name of the column to use as input to the cross encoder model.
-* `return_score` : `str`, default `"relevance"`
-    options are "relevance" or "all". Only "relevance" is supported for now.
-* `api_key` : `str`, default `None`
-    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
-
-
-## Building Custom Rerankers
-You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
-
-The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
-
-```python
-
-from lancedb.rerankers import Reranker
-import pyarrow as pa
-
-class MyReranker(Reranker):
-    def __init__(self, param1, param2, ..., return_score="relevance"):
-        super().__init__(return_score)
-        self.param1 = param1
-        self.param2 = param2
-    
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
-        # Use the built-in merging function
-        combined_result = self.merge_results(vector_results, fts_results)
-        
-        # Do something with the combined results
-        # ...
-
-        # Return the combined results
-        return combined_result
-
-```
-
-### Example of a Custom Reranker
-For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
-
-```python
-
-from typing import List, Union
-import pandas as pd
-from lancedb.rerankers import CohereReranker
-
-class MofidifiedCohereReranker(CohereReranker):
-    def __init__(self, filters: Union[str, List[str]], **kwargs):
-        super().__init__(**kwargs)
-        filters = filters if isinstance(filters, list) else [filters]
-        self.filters = filters
-
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
-        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
-        df = combined_result.to_pandas()
-        for filter in self.filters:
-            df = df.query("not text.str.contains(@filter)")
-
-        return pa.Table.from_pandas(df)
-
-```
-
-!!! tip
-    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/index.md

@@ -28,7 +28,7 @@ LanceDB **Cloud** is a SaaS (software-as-a-service) solution that runs serverles
 
 * Fast production-scale vector similarity, full-text & hybrid search and a SQL query interface (via [DataFusion](https://github.com/apache/arrow-datafusion))
 
-* Python, Javascript/Typescript, and Rust support
+* Native Python and Javascript/Typescript support
 
 * Store, query & manage multi-modal data (text, images, videos, point clouds, etc.), not just the embeddings and metadata
 
@@ -54,4 +54,3 @@ The following pages go deeper into the internal of LanceDB and how to use it.
 * [Ecosystem Integrations](integrations/index.md): Integrate LanceDB with other tools in the data ecosystem
 * [Python API Reference](python/python.md): Python OSS and Cloud API references
 * [JavaScript API Reference](javascript/modules.md): JavaScript OSS and Cloud API references
-* [Rust API Reference](https://docs.rs/lancedb/latest/lancedb/index.html): Rust API reference

docs/src/integrations/index.md

@@ -13,7 +13,7 @@ Get started using these examples and quick links.
 | Integrations | |
 |---|---:|
 | <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://lancedb.github.io/lancedb/integrations/langchain/) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
 | <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/integrations/langchain.md

@@ -1,92 +0,0 @@
-# Langchain
-![Illustration](../assets/langchain.png)
-
-## Quick Start
-You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
-```python
-import os
-from langchain.document_loaders import TextLoader
-from langchain.vectorstores import LanceDB
-from langchain_openai import OpenAIEmbeddings
-from langchain_text_splitters import CharacterTextSplitter
-
-os.environ["OPENAI_API_KEY"] = "sk-..."
-
-loader = TextLoader("../../modules/state_of_the_union.txt") # Replace with your data path
-documents = loader.load()
-
-documents = CharacterTextSplitter().split_documents(documents)
-embeddings = OpenAIEmbeddings()
-
-docsearch = LanceDB.from_documents(documents, embeddings)
-query = "What did the president say about Ketanji Brown Jackson"
-docs = docsearch.similarity_search(query)
-print(docs[0].page_content)
-```
-
-## Documentation
-In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
-You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  
-
-The exhaustive list of parameters for `LanceDB` vector store are :  
-- `connection`: (Optional) `lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.  
-- `embedding`: Langchain embedding model.  
-- `vector_key`: (Optional) Column name to use for vector's in the table. Defaults to `'vector'`.   
-- `id_key`: (Optional) Column name to use for id's in the table. Defaults to `'id'`.  
-- `text_key`: (Optional) Column name to use for text in the table. Defaults to `'text'`.  
-- `table_name`: (Optional) Name of your table in the database. Defaults to `'vectorstore'`.  
-- `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
-- `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
-- `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
-
-```python
-db_url = "db://lang_test" # url of db you created
-api_key = "xxxxx" # your API key
-region="us-east-1-dev"  # your selected region
-
-vector_store = LanceDB(
-    uri=db_url,
-    api_key=api_key, #(dont include for local API)
-    region=region, #(dont include for local API)
-    embedding=embeddings,
-    table_name='langchain_test' #Optional
-    )
-```
-
-### Methods 
-To add texts and store respective embeddings automatically:   
-##### add_texts()
-- `texts`: `Iterable` of strings to add to the vectorstore.
-- `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
-- `ids`: Optional `list` of ids to associate with the texts. 
-
-
-```python
-vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
-
-#Additionaly, to explore the table you can load it into a df or save it in a csv file:
-
-tbl = vector_store.get_table()
-print("tbl:", tbl)
-pd_df = tbl.to_pandas()
-pd_df.to_csv("docsearch.csv", index=False)
-
-# you can also create a new vector store object using an older connection object:
-vector_store = LanceDB(connection=tbl, embedding=embeddings)
-```
-For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
-##### create_index() 
-- `col_name`: `Optional[str] = None`
-- `vector_col`: `Optional[str] = None`
-- `num_partitions`: `Optional[int] = 256`
-- `num_sub_vectors`: `Optional[int] = 96`
-- `index_cache_size`: `Optional[int] = None`
-
-```python
-# for creating vector index
-vector_store.create_index(vector_col='vector', metric = 'cosine')
-
-# for creating scalar index(for non-vector columns)
-vector_store.create_index(col_name='text')
-
-```

docs/src/integrations/llamaIndex.md

@@ -1,139 +0,0 @@
-# Llama-Index
-![Illustration](../assets/llama-index.jpg)
-
-## Quick start
-You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. You can run the below script to try it out :
-```python
-import logging
-import sys
-
-# Uncomment to see debug logs
-# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
-
-from llama_index.core import SimpleDirectoryReader, Document, StorageContext
-from llama_index.core import VectorStoreIndex
-from llama_index.vector_stores.lancedb import LanceDBVectorStore
-import textwrap
-import openai
-
-openai.api_key = "sk-..."
-
-documents = SimpleDirectoryReader("./data/your-data-dir/").load_data()
-print("Document ID:", documents[0].doc_id, "Document Hash:", documents[0].hash)
-
-## For LanceDB cloud :
-# vector_store = LanceDBVectorStore( 
-#     uri="db://db_name", # your remote DB URI
-#     api_key="sk_..", # lancedb cloud api key
-#     region="your-region" # the region you configured
-#     ...
-# )
-
-vector_store = LanceDBVectorStore(
-    uri="./lancedb", mode="overwrite", query_type="vector"
-)
-storage_context = StorageContext.from_defaults(vector_store=vector_store)
-
-index = VectorStoreIndex.from_documents(
-    documents, storage_context=storage_context
-)
-lance_filter = "metadata.file_name = 'paul_graham_essay.txt' "
-retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
-response = retriever.retrieve("What did the author do growing up?")
-```
-
-### Filtering
-For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
-```python
-from llama_index.core.vector_stores import (
-    MetadataFilters,
-    FilterOperator,
-    FilterCondition,
-    MetadataFilter,
-)
-
-query_filters = MetadataFilters(
-    filters=[
-        MetadataFilter(
-            key="creation_date", operator=FilterOperator.EQ, value="2024-05-23"
-        ),
-        MetadataFilter(
-            key="file_size", value=75040, operator=FilterOperator.GT
-        ),
-    ],
-    condition=FilterCondition.AND,
-)
-```
-
-### Hybrid Search
-For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose.
-```python
-from lancedb.rerankers import ColbertReranker
-
-reranker = ColbertReranker()
-vector_store._add_reranker(reranker)
-
-query_engine = index.as_query_engine(
-    filters=query_filters,
-    vector_store_kwargs={
-        "query_type": "hybrid",
-    }
-)
-
-response = query_engine.query("How much did Viaweb charge per month?")
-```
-
-In the above snippet, you can change/specify query_type again when creating the engine/retriever.
-
-## API reference
-The exhaustive list of parameters for `LanceDBVectorStore` vector store are :  
-- `connection`: Optional, `lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.
-- `uri`: Optional[str], the uri of your database. Defaults to `"/tmp/lancedb"`.
-- `table_name` : Optional[str], Name of your table in the database. Defaults to `"vectors"`.
-- `table`: Optional[Any], `lancedb.db.LanceTable` object to be passed. Defaults to `None`. 
-- `vector_column_name`: Optional[Any], Column name to use for vector's in the table. Defaults to `'vector'`.   
-- `doc_id_key`: Optional[str], Column name to use for document id's in the table. Defaults to `'doc_id'`.  
-- `text_key`: Optional[str], Column name to use for text in the table. Defaults to `'text'`.  
-- `api_key`: Optional[str], API key to use for LanceDB cloud database. Defaults to `None`.  
-- `region`: Optional[str], Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
-- `nprobes` : Optional[int], Set the number of probes to use. Only applicable if ANN index is created on the table else its ignored. Defaults to `20`.
-- `refine_factor` : Optional[int], Refine the results by reading extra elements and re-ranking them in memory. Defaults to `None`.
-- `reranker`: Optional[Any], The reranker to use for LanceDB.
-        Defaults to `None`.
-- `overfetch_factor`: Optional[int], The factor by which to fetch more results.
-        Defaults to `1`.
-- `mode`: Optional[str], The mode to use for LanceDB.
-            Defaults to `"overwrite"`.
-- `query_type`:Optional[str], The type of query to use for LanceDB.
-            Defaults to `"vector"`.
-
-
-### Methods 
-
-- __from_table(cls, table: lancedb.db.LanceTable) -> `LanceDBVectorStore`__ : (class method) Creates instance from lancedb table. 
-
-- **_add_reranker(self, reranker: lancedb.rerankers.Reranker) -> `None`** : Add a reranker to an existing vector store. 
-    - Usage :
-        ```python
-        from lancedb.rerankers import ColbertReranker
-        reranker = ColbertReranker()
-        vector_store._add_reranker(reranker)
-        ```               
-- **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
-- __create_index(  
-  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
-) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
-        Make sure your vector column has enough data before creating an index on it.
-
-- __add(self, nodes: `List[BaseNode]`, **add_kwargs: `Any`, ) -> `List[str]`__ :
-adds Nodes to the table
-
-- **delete(self, ref_doc_id: `str`) -> `None`**: Delete nodes using with node_ids.
-- **delete_nodes(self, node_ids: `List[str]`) -> `None`** : Delete nodes using with node_ids.
-- __query(
-        self,
-        query: `VectorStoreQuery`,
-        **kwargs: `Any`,
-    ) -> `VectorStoreQueryResult`__:
-        Query index(`VectorStoreIndex`) for top k most similar nodes. Accepts llamaIndex `VectorStoreQuery` object.

docs/src/javascript.md

@@ -0,0 +1,62 @@
+# Javascript API Reference
+
+This section contains the API reference for LanceDB Javascript API.
+
+## Installation
+
+```bash
+npm install vectordb
+```
+
+This will download the appropriate native library for your platform. We currently
+support:
+
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or arch64 Windows.
+
+## Usage
+
+### Basic Example
+Connect to a local directory
+```javascript
+const lancedb = require('vectordb');
+//connect to a local database
+const db = await lancedb.connect('data/sample-lancedb');
+```
+Connect to LancdDB cloud
+```javascript
+connect to LanceDB Cloud
+const db = await lancedb.connect({
+    uri: "db://my-database",
+    apiKey: "sk_...",
+    region: "us-east-1"
+});
+```
+Create a table followed by a search
+```javascript
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
+console.log(results);
+```
+
+The [examples](./examples) folder contains complete examples.
+
+## Table of contents
+### Connection
+Connect to a LanceDB database.
+
+- [Connection](interfaces/Connection.md)
+### Table
+A Table is a collection of Records in a LanceDB Database.
+
+- [Table](interfaces/Table.md)
+### Query
+The LanceDB Query
+
+- [Query](classes/Query.md)
+

docs/src/javascript/classes/DefaultWriteOptions.md

@@ -38,4 +38,4 @@ A [WriteMode](../enums/WriteMode.md) to use on this operation
 
 #### Defined in
 
-[index.ts:1019](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1019)
+[index.ts:1070](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1070)

docs/src/javascript/classes/LocalConnection.md

@@ -46,7 +46,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:489](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L489)
+[index.ts:496](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L496)
 
 ## Properties
 
@@ -56,7 +56,7 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:487](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L487)
+[index.ts:494](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L494)
 
 ___
 
@@ -74,7 +74,7 @@ ___
 
 #### Defined in
 
-[index.ts:486](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L486)
+[index.ts:493](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L493)
 
 ## Accessors
 
@@ -92,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:494](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L494)
+[index.ts:501](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L501)
 
 ## Methods
 
@@ -113,7 +113,7 @@ Creates a new Table, optionally initializing it with new data.
 | Name | Type |
 | :------ | :------ |
 | `name` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
-| `data?` | `Record`\<`string`, `unknown`\>[] |
+| `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
 | `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
 | `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
 
@@ -127,7 +127,7 @@ Creates a new Table, optionally initializing it with new data.
 
 #### Defined in
 
-[index.ts:542](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L542)
+[index.ts:549](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L549)
 
 ___
 
@@ -158,7 +158,7 @@ ___
 
 #### Defined in
 
-[index.ts:576](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L576)
+[index.ts:583](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L583)
 
 ___
 
@@ -184,7 +184,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:630](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L630)
+[index.ts:637](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L637)
 
 ___
 
@@ -210,7 +210,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:510](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L510)
+[index.ts:517](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L517)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -239,7 +239,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:518](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L518)
+[index.ts:525](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L525)
 
 ▸ **openTable**\<`T`\>(`name`, `embeddings?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
 
@@ -266,7 +266,7 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:522](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L522)
+[index.ts:529](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L529)
 
 ___
 
@@ -286,4 +286,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:501](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L501)
+[index.ts:508](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L508)

docs/src/javascript/classes/LocalTable.md

@@ -74,7 +74,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:642](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L642)
+[index.ts:649](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L649)
 
 • **new LocalTable**\<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
@@ -95,7 +95,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:649](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L649)
+[index.ts:656](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L656)
 
 ## Properties
 
@@ -105,7 +105,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:639](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L639)
+[index.ts:646](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L646)
 
 ___
 
@@ -115,7 +115,7 @@ ___
 
 #### Defined in
 
-[index.ts:638](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L638)
+[index.ts:645](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L645)
 
 ___
 
@@ -125,7 +125,7 @@ ___
 
 #### Defined in
 
-[index.ts:637](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L637)
+[index.ts:644](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L644)
 
 ___
 
@@ -143,7 +143,7 @@ ___
 
 #### Defined in
 
-[index.ts:640](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L640)
+[index.ts:647](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L647)
 
 ___
 
@@ -153,7 +153,7 @@ ___
 
 #### Defined in
 
-[index.ts:636](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L636)
+[index.ts:643](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L643)
 
 ___
 
@@ -179,7 +179,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:688](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L688)
+[index.ts:695](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L695)
 
 ## Accessors
 
@@ -197,7 +197,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:668](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L668)
+[index.ts:675](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L675)
 
 ___
 
@@ -215,7 +215,7 @@ ___
 
 #### Defined in
 
-[index.ts:849](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L849)
+[index.ts:875](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L875)
 
 ## Methods
 
@@ -229,7 +229,7 @@ Insert records into this Table.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
 
 #### Returns
 
@@ -243,7 +243,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:696](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L696)
+[index.ts:703](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L703)
 
 ___
 
@@ -257,7 +257,7 @@ ___
 
 #### Defined in
 
-[index.ts:861](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L861)
+[index.ts:887](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L887)
 
 ___
 
@@ -267,6 +267,8 @@ ___
 
 Clean up old versions of the table, freeing disk space.
 
+Note: this API is not yet available on LanceDB Cloud
+
 #### Parameters
 
 | Name | Type | Description |
@@ -280,7 +282,7 @@ Clean up old versions of the table, freeing disk space.
 
 #### Defined in
 
-[index.ts:808](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L808)
+[index.ts:833](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L833)
 
 ___
 
@@ -293,6 +295,8 @@ Run the compaction process on the table.
 This can be run after making several small appends to optimize the table
 for faster reads.
 
+Note: this API is not yet available on LanceDB Cloud
+
 #### Parameters
 
 | Name | Type | Description |
@@ -307,7 +311,7 @@ Metrics about the compaction operation.
 
 #### Defined in
 
-[index.ts:831](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L831)
+[index.ts:857](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L857)
 
 ___
 
@@ -327,7 +331,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:749](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L749)
+[index.ts:773](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L773)
 
 ___
 
@@ -357,7 +361,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:734](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L734)
+[index.ts:758](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L758)
 
 ___
 
@@ -392,7 +396,7 @@ await table.createScalarIndex('my_col')
 
 #### Defined in
 
-[index.ts:742](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L742)
+[index.ts:766](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L766)
 
 ___
 
@@ -418,7 +422,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:758](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L758)
+[index.ts:782](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L782)
 
 ___
 
@@ -440,7 +444,7 @@ Creates a filter query to find all rows matching the specified criteria
 
 #### Defined in
 
-[index.ts:684](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L684)
+[index.ts:691](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L691)
 
 ___
 
@@ -454,7 +458,7 @@ ___
 
 #### Defined in
 
-[index.ts:854](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L854)
+[index.ts:880](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L880)
 
 ___
 
@@ -480,7 +484,7 @@ Get statistics about an index.
 
 #### Defined in
 
-[index.ts:845](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L845)
+[index.ts:871](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L871)
 
 ___
 
@@ -500,7 +504,7 @@ List the indicies on this table.
 
 #### Defined in
 
-[index.ts:841](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L841)
+[index.ts:867](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L867)
 
 ___
 
@@ -514,7 +518,7 @@ Insert records into this Table, replacing its contents.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table Type Table is ArrowTable |
 
 #### Returns
 
@@ -528,7 +532,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:716](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L716)
+[index.ts:732](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L732)
 
 ___
 
@@ -554,7 +558,7 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:676](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L676)
+[index.ts:683](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L683)
 
 ___
 
@@ -580,4 +584,4 @@ Update rows in this table.
 
 #### Defined in
 
-[index.ts:771](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L771)
+[index.ts:795](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L795)

docs/src/javascript/classes/MakeArrowTableOptions.md

@@ -0,0 +1,56 @@
+[vectordb](../README.md) / [Exports](../modules.md) / MakeArrowTableOptions
+
+# Class: MakeArrowTableOptions
+
+Options to control the makeArrowTable call.
+
+## Table of contents
+
+### Constructors
+
+- [constructor](MakeArrowTableOptions.md#constructor)
+
+### Properties
+
+- [schema](MakeArrowTableOptions.md#schema)
+- [vectorColumns](MakeArrowTableOptions.md#vectorcolumns)
+
+## Constructors
+
+### constructor
+
+• **new MakeArrowTableOptions**(`values?`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `values?` | `Partial`\<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)\> |
+
+#### Defined in
+
+[arrow.ts:56](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/arrow.ts#L56)
+
+## Properties
+
+### schema
+
+• `Optional` **schema**: `Schema`\<`any`\>
+
+Provided schema.
+
+#### Defined in
+
+[arrow.ts:49](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/arrow.ts#L49)
+
+___
+
+### vectorColumns
+
+• **vectorColumns**: `Record`\<`string`, `VectorColumnOptions`\>
+
+Vector columns
+
+#### Defined in
+
+[arrow.ts:52](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/arrow.ts#L52)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:22](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/embedding/openai.ts#L22)
 
 ## Properties
 
@@ -50,17 +50,17 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:20](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/embedding/openai.ts#L20)
 
 ___
 
 ### \_openai
 
-• `Private` `Readonly` **\_openai**: `any`
+• `Private` `Readonly` **\_openai**: `OpenAI`
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:56](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/embedding/openai.ts#L56)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:43](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/embedding/openai.ts#L43)

docs/src/javascript/classes/Query.md

@@ -65,7 +65,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:38](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L38)
+[query.ts:38](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L38)
 
 ## Properties
 
@@ -75,7 +75,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[query.ts:36](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L36)
+[query.ts:36](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L36)
 
 ___
 
@@ -85,7 +85,7 @@ ___
 
 #### Defined in
 
-[query.ts:33](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L33)
+[query.ts:33](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L33)
 
 ___
 
@@ -95,7 +95,7 @@ ___
 
 #### Defined in
 
-[query.ts:29](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L29)
+[query.ts:29](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L29)
 
 ___
 
@@ -105,7 +105,7 @@ ___
 
 #### Defined in
 
-[query.ts:34](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L34)
+[query.ts:34](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L34)
 
 ___
 
@@ -115,7 +115,7 @@ ___
 
 #### Defined in
 
-[query.ts:31](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L31)
+[query.ts:31](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L31)
 
 ___
 
@@ -125,7 +125,7 @@ ___
 
 #### Defined in
 
-[query.ts:35](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L35)
+[query.ts:35](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L35)
 
 ___
 
@@ -135,7 +135,7 @@ ___
 
 #### Defined in
 
-[query.ts:26](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L26)
+[query.ts:26](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L26)
 
 ___
 
@@ -145,7 +145,7 @@ ___
 
 #### Defined in
 
-[query.ts:28](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L28)
+[query.ts:28](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L28)
 
 ___
 
@@ -155,7 +155,7 @@ ___
 
 #### Defined in
 
-[query.ts:30](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L30)
+[query.ts:30](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L30)
 
 ___
 
@@ -165,7 +165,7 @@ ___
 
 #### Defined in
 
-[query.ts:32](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L32)
+[query.ts:32](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L32)
 
 ___
 
@@ -175,7 +175,7 @@ ___
 
 #### Defined in
 
-[query.ts:27](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L27)
+[query.ts:27](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L27)
 
 ___
 
@@ -201,7 +201,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:87](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L87)
+[query.ts:87](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L87)
 
 ## Methods
 
@@ -223,7 +223,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[query.ts:115](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L115)
+[query.ts:115](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L115)
 
 ___
 
@@ -245,7 +245,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[query.ts:82](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L82)
+[query.ts:82](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L82)
 
 ___
 
@@ -259,7 +259,7 @@ ___
 
 #### Defined in
 
-[query.ts:142](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L142)
+[query.ts:143](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L143)
 
 ___
 
@@ -281,7 +281,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[query.ts:55](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L55)
+[query.ts:55](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L55)
 
 ___
 
@@ -307,7 +307,7 @@ MetricType for the different options
 
 #### Defined in
 
-[query.ts:102](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L102)
+[query.ts:102](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L102)
 
 ___
 
@@ -329,7 +329,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[query.ts:73](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L73)
+[query.ts:73](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L73)
 
 ___
 
@@ -349,7 +349,7 @@ ___
 
 #### Defined in
 
-[query.ts:107](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L107)
+[query.ts:107](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L107)
 
 ___
 
@@ -371,7 +371,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[query.ts:64](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L64)
+[query.ts:64](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L64)
 
 ___
 
@@ -393,4 +393,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[query.ts:93](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/query.ts#L93)
+[query.ts:93](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/query.ts#L93)

docs/src/javascript/classes/RemoteConnection.md

@@ -0,0 +1,224 @@
+[vectordb](../README.md) / [Exports](../modules.md) / RemoteConnection
+
+# Class: RemoteConnection
+
+Remote connection.
+
+## Implements
+
+- [`Connection`](../interfaces/Connection.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](RemoteConnection.md#constructor)
+
+### Properties
+
+- [\_client](RemoteConnection.md#_client)
+- [\_dbName](RemoteConnection.md#_dbname)
+
+### Accessors
+
+- [uri](RemoteConnection.md#uri)
+
+### Methods
+
+- [createTable](RemoteConnection.md#createtable)
+- [dropTable](RemoteConnection.md#droptable)
+- [openTable](RemoteConnection.md#opentable)
+- [tableNames](RemoteConnection.md#tablenames)
+
+## Constructors
+
+### constructor
+
+• **new RemoteConnection**(`opts`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `opts` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
+
+#### Defined in
+
+[remote/index.ts:48](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L48)
+
+## Properties
+
+### \_client
+
+• `Private` `Readonly` **\_client**: `HttpLancedbClient`
+
+#### Defined in
+
+[remote/index.ts:45](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L45)
+
+___
+
+### \_dbName
+
+• `Private` `Readonly` **\_dbName**: `string`
+
+#### Defined in
+
+[remote/index.ts:46](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L46)
+
+## Accessors
+
+### uri
+
+• `get` **uri**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
+
+#### Defined in
+
+[remote/index.ts:75](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L75)
+
+## Methods
+
+### createTable
+
+▸ **createTable**\<`T`\>(`nameOrOpts`, `data?`, `optsOrEmbedding?`, `opt?`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+Creates a new Table, optionally initializing it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `nameOrOpts` | `string` \| [`CreateTableOptions`](../interfaces/CreateTableOptions.md)\<`T`\> |
+| `data?` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] |
+| `optsOrEmbedding?` | [`WriteOptions`](../interfaces/WriteOptions.md) \| [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+| `opt?` | [`WriteOptions`](../interfaces/WriteOptions.md) |
+
+#### Returns
+
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
+
+#### Defined in
+
+[remote/index.ts:107](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L107)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`\<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[remote/index.ts:175](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L175)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`\<[`Table`](../interfaces/Table.md)\<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[remote/index.ts:91](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L91)
+
+▸ **openTable**\<`T`\>(`name`, `embeddings`): `Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+
+#### Returns
+
+`Promise`\<[`Table`](../interfaces/Table.md)\<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[remote/index.ts:92](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L92)
+
+___
+
+### tableNames
+
+▸ **tableNames**(`pageToken?`, `limit?`): `Promise`\<`string`[]\>
+
+#### Parameters
+
+| Name | Type | Default value |
+| :------ | :------ | :------ |
+| `pageToken` | `string` | `''` |
+| `limit` | `number` | `10` |
+
+#### Returns
+
+`Promise`\<`string`[]\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
+
+#### Defined in
+
+[remote/index.ts:80](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L80)

docs/src/javascript/classes/RemoteTable.md

@@ -0,0 +1,470 @@
+[vectordb](../README.md) / [Exports](../modules.md) / RemoteTable
+
+# Class: RemoteTable\<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implements
+
+- [`Table`](../interfaces/Table.md)\<`T`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](RemoteTable.md#constructor)
+
+### Properties
+
+- [\_client](RemoteTable.md#_client)
+- [\_embeddings](RemoteTable.md#_embeddings)
+- [\_name](RemoteTable.md#_name)
+
+### Accessors
+
+- [name](RemoteTable.md#name)
+- [schema](RemoteTable.md#schema)
+
+### Methods
+
+- [add](RemoteTable.md#add)
+- [countRows](RemoteTable.md#countrows)
+- [createIndex](RemoteTable.md#createindex)
+- [createScalarIndex](RemoteTable.md#createscalarindex)
+- [delete](RemoteTable.md#delete)
+- [indexStats](RemoteTable.md#indexstats)
+- [listIndices](RemoteTable.md#listindices)
+- [overwrite](RemoteTable.md#overwrite)
+- [search](RemoteTable.md#search)
+- [update](RemoteTable.md#update)
+
+## Constructors
+
+### constructor
+
+• **new RemoteTable**\<`T`\>(`client`, `name`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `client` | `HttpLancedbClient` |
+| `name` | `string` |
+
+#### Defined in
+
+[remote/index.ts:234](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L234)
+
+• **new RemoteTable**\<`T`\>(`client`, `name`, `embeddings`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `client` | `HttpLancedbClient` |
+| `name` | `string` |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\> |
+
+#### Defined in
+
+[remote/index.ts:235](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L235)
+
+## Properties
+
+### \_client
+
+• `Private` `Readonly` **\_client**: `HttpLancedbClient`
+
+#### Defined in
+
+[remote/index.ts:230](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L230)
+
+___
+
+### \_embeddings
+
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)\<`T`\>
+
+#### Defined in
+
+[remote/index.ts:231](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L231)
+
+___
+
+### \_name
+
+• `Private` `Readonly` **\_name**: `string`
+
+#### Defined in
+
+[remote/index.ts:232](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L232)
+
+## Accessors
+
+### name
+
+• `get` **name**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
+
+#### Defined in
+
+[remote/index.ts:250](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L250)
+
+___
+
+### schema
+
+• `get` **schema**(): `Promise`\<`any`\>
+
+#### Returns
+
+`Promise`\<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[schema](../interfaces/Table.md#schema)
+
+#### Defined in
+
+[remote/index.ts:254](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L254)
+
+## Methods
+
+### add
+
+▸ **add**(`data`): `Promise`\<`number`\>
+
+Insert records into this Table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`\<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
+
+#### Defined in
+
+[remote/index.ts:273](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L273)
+
+___
+
+### countRows
+
+▸ **countRows**(): `Promise`\<`number`\>
+
+Returns the number of rows in this table.
+
+#### Returns
+
+`Promise`\<`number`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
+
+#### Defined in
+
+[remote/index.ts:372](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L372)
+
+___
+
+### createIndex
+
+▸ **createIndex**(`indexParams`): `Promise`\<`void`\>
+
+Create an ANN index on this Table vector index.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+**`See`**
+
+VectorIndexParams.
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
+
+#### Defined in
+
+[remote/index.ts:326](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L326)
+
+___
+
+### createScalarIndex
+
+▸ **createScalarIndex**(`column`, `replace`): `Promise`\<`void`\>
+
+Create a scalar index on this Table for the given column
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `column` | `string` | The column to index |
+| `replace` | `boolean` | If false, fail if an index already exists on the column Scalar indices, like vector indices, can be used to speed up scans. A scalar index can speed up scans that contain filter expressions on the indexed column. For example, the following scan will be faster if the column `my_col` has a scalar index: ```ts const con = await lancedb.connect('./.lancedb'); const table = await con.openTable('images'); const results = await table.where('my_col = 7').execute(); ``` Scalar indices can also speed up scans containing a vector search and a prefilter: ```ts const con = await lancedb.connect('././lancedb'); const table = await con.openTable('images'); const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true); ``` Scalar indices can only speed up scans for basic filters using equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set membership (e.g. `my_col IN (0, 1, 2)`) Scalar indices can be used if the filter contains multiple indexed columns and the filter criteria are AND'd or OR'd together (e.g. `my_col < 0 AND other_col> 100`) Scalar indices may be used if the filter contains non-indexed columns but, depending on the structure of the filter, they may not be usable. For example, if the column `not_indexed` does not have a scalar index then the filter `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on `my_col`. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect('././lancedb')
+const table = await con.openTable('images')
+await table.createScalarIndex('my_col')
+```
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createScalarIndex](../interfaces/Table.md#createscalarindex)
+
+#### Defined in
+
+[remote/index.ts:368](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L368)
+
+___
+
+### delete
+
+▸ **delete**(`filter`): `Promise`\<`void`\>
+
+Delete rows from this table.
+
+This can be used to delete a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [1, 2]},
+   {id: 2, vector: [3, 4]},
+   {id: 3, vector: [5, 6]},
+];
+const tbl = await con.createTable("my_table", data)
+await tbl.delete("id = 2")
+await tbl.countRows() // Returns 2
+```
+
+If you have a list of values to delete, you can combine them into a
+stringified list and use the `IN` operator:
+
+```ts
+const to_remove = [1, 5];
+await tbl.delete(`id IN (${to_remove.join(",")})`)
+await tbl.countRows() // Returns 1
+```
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
+
+#### Defined in
+
+[remote/index.ts:377](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L377)
+
+___
+
+### indexStats
+
+▸ **indexStats**(`indexUuid`): `Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+Get statistics about an index.
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `indexUuid` | `string` |
+
+#### Returns
+
+`Promise`\<[`IndexStats`](../interfaces/IndexStats.md)\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[indexStats](../interfaces/Table.md#indexstats)
+
+#### Defined in
+
+[remote/index.ts:414](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L414)
+
+___
+
+### listIndices
+
+▸ **listIndices**(): `Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+List the indicies on this table.
+
+#### Returns
+
+`Promise`\<[`VectorIndex`](../interfaces/VectorIndex.md)[]\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[listIndices](../interfaces/Table.md#listindices)
+
+#### Defined in
+
+[remote/index.ts:403](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L403)
+
+___
+
+### overwrite
+
+▸ **overwrite**(`data`): `Promise`\<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`\<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
+
+#### Defined in
+
+[remote/index.ts:300](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L300)
+
+___
+
+### search
+
+▸ **search**(`query`): [`Query`](Query.md)\<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+#### Returns
+
+[`Query`](Query.md)\<`T`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
+
+#### Defined in
+
+[remote/index.ts:269](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L269)
+
+___
+
+### update
+
+▸ **update**(`args`): `Promise`\<`void`\>
+
+Update rows in this table.
+
+This can be used to update a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `args` | [`UpdateArgs`](../interfaces/UpdateArgs.md) \| [`UpdateSqlArgs`](../interfaces/UpdateSqlArgs.md) | see [UpdateArgs](../interfaces/UpdateArgs.md) and [UpdateSqlArgs](../interfaces/UpdateSqlArgs.md) for more details |
+
+#### Returns
+
+`Promise`\<`void`\>
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [3, 3], name: 'Ye'},
+   {id: 2, vector: [4, 4], name: 'Mike'},
+];
+const tbl = await con.createTable("my_table", data)
+
+await tbl.update({
+  where: "id = 2",
+  values: { vector: [2, 2], name: "Michael" },
+})
+
+let results = await tbl.search([1, 1]).execute();
+// Returns [
+//   {id: 2, vector: [2, 2], name: 'Michael'}
+//   {id: 1, vector: [3, 3], name: 'Ye'}
+// ]
+```
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[update](../interfaces/Table.md#update)
+
+#### Defined in
+
+[remote/index.ts:383](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/remote/index.ts#L383)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:1041](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1041)
+[index.ts:1092](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1092)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:1046](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1046)
+[index.ts:1097](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1097)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:1036](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1036)
+[index.ts:1087](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1087)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:1007](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1007)
+[index.ts:1058](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1058)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:1003](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1003)
+[index.ts:1054](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1054)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:1005](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L1005)
+[index.ts:1056](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L1056)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -18,7 +18,7 @@
 
 #### Defined in
 
-[index.ts:54](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L54)
+[index.ts:57](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L57)
 
 ___
 
@@ -28,7 +28,7 @@ ___
 
 #### Defined in
 
-[index.ts:56](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L56)
+[index.ts:59](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L59)
 
 ___
 
@@ -38,4 +38,4 @@ ___
 
 #### Defined in
 
-[index.ts:58](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L58)
+[index.ts:61](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L61)

docs/src/javascript/interfaces/CleanupStats.md

@@ -19,7 +19,7 @@ The number of bytes removed from disk.
 
 #### Defined in
 
-[index.ts:878](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L878)
+[index.ts:904](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L904)
 
 ___
 
@@ -31,4 +31,4 @@ The number of old table versions removed.
 
 #### Defined in
 
-[index.ts:882](https://github.com/lancedb/lancedb/blob/c89d5e6/node/src/index.ts#L882)
+[index.ts:908](https://github.com/lancedb/lancedb/blob/5228ca4/node/src/index.ts#L908)