Bump version: 0.12.0-beta.0 → 0.12.0

PR fixes #1514

.bumpversion.cfg

@@ -1,12 +0,0 @@
-[bumpversion]
-current_version = 0.4.14
-commit = True
-message = Bump version: {current_version} → {new_version}
-tag = True
-tag_name = v{new_version}
-
-[bumpversion:file:node/package.json]
-
-[bumpversion:file:rust/ffi/node/Cargo.toml]
-
-[bumpversion:file:rust/lancedb/Cargo.toml]

.bumpversion.toml

@@ -0,0 +1,57 @@
+[tool.bumpversion]
+current_version = "0.8.0"
+parse = """(?x)
+    (?P<major>0|[1-9]\\d*)\\.
+    (?P<minor>0|[1-9]\\d*)\\.
+    (?P<patch>0|[1-9]\\d*)
+    (?:-(?P<pre_l>[a-zA-Z-]+)\\.(?P<pre_n>0|[1-9]\\d*))?
+"""
+serialize = [
+    "{major}.{minor}.{patch}-{pre_l}.{pre_n}",
+    "{major}.{minor}.{patch}",
+]
+search = "{current_version}"
+replace = "{new_version}"
+regex = false
+ignore_missing_version = false
+ignore_missing_files = false
+tag = true
+sign_tags = false
+tag_name = "v{new_version}"
+tag_message = "Bump version: {current_version} → {new_version}"
+allow_dirty = true
+commit = true
+message = "Bump version: {current_version} → {new_version}"
+commit_args = ""
+
+[tool.bumpversion.parts.pre_l]
+values = ["beta", "final"]
+optional_value = "final"
+
+[[tool.bumpversion.files]]
+filename = "node/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+[[tool.bumpversion.files]]
+filename = "nodejs/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+# nodejs binary packages
+[[tool.bumpversion.files]]
+glob = "nodejs/npm/*/package.json"
+search = "\"version\": \"{current_version}\","
+replace = "\"version\": \"{new_version}\","
+
+# Cargo files
+# ------------
+[[tool.bumpversion.files]]
+filename = "rust/ffi/node/Cargo.toml"
+search = "\nversion = \"{current_version}\""
+replace = "\nversion = \"{new_version}\""
+
+[[tool.bumpversion.files]]
+filename = "rust/lancedb/Cargo.toml"
+search = "\nversion = \"{current_version}\""
+replace = "\nversion = \"{new_version}\""

.github/labeler.yml

@@ -0,0 +1,33 @@
+version: 1
+appendOnly: true
+# Labels are applied based on conventional commits standard
+# https://www.conventionalcommits.org/en/v1.0.0/
+# These labels are later used in release notes. See .github/release.yml
+labels:
+# If the PR title has an ! before the : it will be considered a breaking change
+# For example, `feat!: add new feature` will be considered a breaking change
+- label: breaking-change
+  title: "^[^:]+!:.*"
+- label: breaking-change
+  body: "BREAKING CHANGE"
+- label: enhancement
+  title: "^feat(\\(.+\\))?!?:.*"
+- label: bug
+  title: "^fix(\\(.+\\))?!?:.*"
+- label: documentation
+  title: "^docs(\\(.+\\))?!?:.*"
+- label: performance
+  title: "^perf(\\(.+\\))?!?:.*"
+- label: ci
+  title: "^ci(\\(.+\\))?!?:.*"
+- label: chore
+  title: "^(chore|test|build|style)(\\(.+\\))?!?:.*"
+- label: Python
+  files:
+    - "^python\\/.*"
+- label: Rust
+  files:
+    - "^rust\\/.*"
+- label: typescript
+  files:
+    - "^node\\/.*"

.github/release_notes.json

@@ -0,0 +1,41 @@
+{
+    "ignore_labels": ["chore"],
+    "pr_template": "- ${{TITLE}} by @${{AUTHOR}} in ${{URL}}",
+    "categories": [
+        {
+            "title": "## 🏆 Highlights",
+            "labels": ["highlight"]
+        },
+        {
+            "title": "## 🛠 Breaking Changes",
+            "labels": ["breaking-change"]
+        },
+        {
+            "title": "## ⚠️ Deprecations ",
+            "labels": ["deprecation"]
+        },
+        {
+            "title": "## 🎉 New Features",
+            "labels": ["enhancement"]
+        },
+        {
+            "title": "## 🐛 Bug Fixes",
+            "labels": ["bug"]
+        },
+        {
+            "title": "## 📚 Documentation",
+            "labels": ["documentation"]
+        },
+        {
+            "title": "## 🚀 Performance Improvements",
+            "labels": ["performance"]
+        },
+        {
+            "title": "## Other Changes"
+        },
+        {
+            "title": "## 🔧 Build and CI",
+            "labels": ["ci"]
+        }
+    ]
+}

.github/workflows/build_linux_wheel/action.yml

@@ -14,6 +14,10 @@ inputs:
     # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
     required: false
     default: "false"
+  manylinux:
+    description: "The manylinux version to build for"
+    required: false
+    default: "2_17"
 runs:
   using: "composite"
   steps:
@@ -28,7 +32,7 @@ runs:
         command: build
         working-directory: python
         target: x86_64-unknown-linux-gnu
-        manylinux: "2_17"
+        manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
@@ -42,8 +46,9 @@ runs:
       with:
         command: build
         working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         target: aarch64-unknown-linux-gnu
-        manylinux: "2_24"
+        manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e

.github/workflows/build_mac_wheel/action.yml

@@ -21,5 +21,6 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
         interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -26,6 +26,7 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
     - uses: actions/upload-artifact@v3
       with:

.github/workflows/cargo-publish.yml

@@ -1,13 +1,20 @@
 name: Cargo Publish
 
 on:
-  release:
-    types: [ published ]
+  push:
+    tags-ignore:
+      # We don't publish pre-releases for Rust. Crates.io is just a source
+      # distribution, so we don't need to publish pre-releases.
+      - 'v*-beta*'
+      - '*-v*' # for example, python-vX.Y.Z
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
+  # Up-to-date compilers needed for fp16kernels.
+  CC: gcc-12
+  CXX: g++-12
 
 jobs:
   build:

.github/workflows/dev.yml

@@ -0,0 +1,81 @@
+name: PR Checks
+
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize, reopened]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  labeler:
+    permissions:
+      pull-requests: write
+    name: Label PR
+    runs-on: ubuntu-latest
+    steps:
+      - uses: srvaroa/labeler@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  commitlint:
+    permissions:
+      pull-requests: write
+    name: Verify PR title / description conforms to semantic-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+      # These rules are disabled because Github will always ensure there
+      # is a blank line between the title and the body and Github will
+      # word wrap the description field to ensure a reasonable max line
+      # length.
+      - run: npm install @commitlint/config-conventional
+      - run: >
+          echo 'module.exports = {
+            "rules": {
+              "body-max-line-length": [0, "always", Infinity],
+              "footer-max-line-length": [0, "always", Infinity],
+              "body-leading-blank": [0, "always"]
+            }
+          }' > .commitlintrc.js
+      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
+        env:
+          COMMIT_MSG: >
+            ${{ github.event.pull_request.title }}
+
+            ${{ github.event.pull_request.body }}
+      - if: failure()
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const message = `**ACTION NEEDED**
+              
+              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
+
+              The PR title and description are used as the merge commit message.\
+              Please update your PR title and description to match the specification.
+
+              For details on the error please inspect the "PR Title Check" action.
+              `
+            // Get list of current comments
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number
+            });
+            // Check if this job already commented
+            for (const comment of comments) {
+              if (comment.body === message) {
+                return // Already commented
+              }
+            }
+            // Post the comment about Conventional Commits
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: message
+            })
+            core.setFailed(message)

.github/workflows/docs_test.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -56,7 +56,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/java.yml

@@ -0,0 +1,113 @@
+name: Build and Run Java JNI Tests
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - java/**
+  pull_request:
+    paths:
+      - java/**
+      - rust/**
+      - .github/workflows/java.yml
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  CARGO_BUILD_JOBS: "1"
+jobs:
+  linux-build-java-11:
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04 + Java 11
+    defaults:
+      run:
+        working-directory: ./java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: java/core/lancedb-jni
+      - name: Run cargo fmt
+        run: cargo fmt --check
+        working-directory: ./java/core/lancedb-jni
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install Java 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+          cache: "maven"
+      - name: Java Style Check
+        run: mvn checkstyle:check
+      # Disable because of issues in lancedb rust core code
+      # - name: Rust Clippy
+      #   working-directory: java/core/lancedb-jni
+      #   run: cargo clippy --all-targets -- -D warnings
+      - name: Running tests with Java 11
+        run: mvn clean test
+  linux-build-java-17:
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04 + Java 17
+    defaults:
+      run:
+        working-directory: ./java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: java/core/lancedb-jni
+      - name: Run cargo fmt
+        run: cargo fmt --check
+        working-directory: ./java/core/lancedb-jni
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: "maven"
+      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
+      - name: Java Style Check
+        run: mvn checkstyle:check
+      # Disable because of issues in lancedb rust core code
+      # - name: Rust Clippy
+      #   working-directory: java/core/lancedb-jni
+      #   run: cargo clippy --all-targets -- -D warnings
+      - name: Running tests with Java 17
+        run: |
+          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
+          -XX:+IgnoreUnrecognizedVMOptions \
+          --add-opens=java.base/java.lang=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
+          --add-opens=java.base/java.io=ALL-UNNAMED \
+          --add-opens=java.base/java.net=ALL-UNNAMED \
+          --add-opens=java.base/java.nio=ALL-UNNAMED \
+          --add-opens=java.base/java.util=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
+          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
+          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
+          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
+          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
+          -Djdk.reflect.useDirectMethodHandle=false \
+          -Dio.netty.tryReflectionSetAccessible=true"
+          JAVA_HOME=$JAVA_17 mvn clean test
+  

.github/workflows/make-release-commit.yml

@@ -1,37 +1,62 @@
 name: Create release commit
 
+# This workflow increments versions, tags the version, and pushes it.
+# When a tag is pushed, another workflow is triggered that creates a GH release
+# and uploads the binaries. This workflow is only for creating the tag.
+
+# This script will enforce that a minor version is incremented if there are any
+# breaking changes since the last minor increment. However, it isn't able to
+# differentiate between breaking changes in Node versus Python. If you wish to
+# bypass this check, you can manually increment the version and push the tag.
 on:
   workflow_dispatch:
     inputs:
       dry_run:
         description: 'Dry run (create the local commit/tags but do not push it)'
         required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
+        default: false
+        type: boolean
+      type:
         description: 'What kind of release is this?'
         required: true
-        default: 'patch'
+        default: 'preview'
         type: choice
         options:
-          - patch
-          - minor
-          - major
+          - preview
+          - stable
+      python:
+        description: 'Make a Python release'
+        required: true
+        default: true
+        type: boolean
+      other:
+        description: 'Make a Node/Rust release'
+        required: true
+        default: true
+        type: boolean
+      bump-minor:
+        description: 'Bump minor version'
+        required: true
+        default: false
+        type: boolean
 
 jobs:
-  bump-version:
+  make-release:
+    # Creates tag and GH release. The GH release will trigger the build and release jobs.
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
-      - name: Check out main
-        uses: actions/checkout@v4
+      - name: Output Inputs
+        run: echo "${{ toJSON(github.event.inputs) }}"
+      - uses: actions/checkout@v4
         with:
-          ref: main
-          persist-credentials: false
           fetch-depth: 0
           lfs: true
+          # It's important we use our token here, as the default token will NOT
+          # trigger any workflows watching for new tags. See:
+          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
       - name: Set git configs for bumpversion
         shell: bash
         run: |
@@ -41,19 +66,34 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: Bump version, create tag and commit
+      - name: Bump Python version
+        if: ${{ inputs.python }}
+        working-directory: python
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          pip install bump2version
-          bumpversion --verbose ${{ inputs.part }}
-      - name: Push new version and tag
-        if: ${{ inputs.dry_run }} == "false"
+          # Need to get the commit before bumping the version, so we can
+          # determine if there are breaking changes in the next step as well.
+          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+          pip install bump-my-version PyGithub packaging
+          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
+      - name: Bump Node/Rust version
+        if: ${{ inputs.other }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pip install bump-my-version PyGithub packaging
+          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
+      - name: Push new version tag
+        if: ${{ !inputs.dry_run }}
         uses: ad-m/github-push-action@master
         with:
+          # Need to use PAT here too to trigger next workflow. See comment above.
           github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-          branch: main
+          branch: ${{ github.ref }}
           tags: true
       - uses: ./.github/workflows/update_package_lock
-        if: ${{ inputs.dry_run }} == "false"
+        if: ${{ !inputs.dry_run && inputs.other }}
         with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/node.yml

@@ -107,6 +107,7 @@ jobs:
       AWS_ENDPOINT: http://localhost:4566
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
+      ALLOW_HTTP: true
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/nodejs.yml

@@ -28,6 +28,10 @@ jobs:
       run:
         shell: bash
         working-directory: nodejs
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -48,8 +52,7 @@ jobs:
         cargo fmt --all -- --check
         cargo clippy --all --all-features -- -D warnings
         npm ci
-        npm run lint
-        npm run chkformat
+        npm run lint-ci
   linux:
     name: Linux (NodeJS ${{ matrix.node-version }})
     timeout-minutes: 30
@@ -81,7 +84,12 @@ jobs:
       run: |
         npm ci
         npm run build
+    - name: Setup localstack
+      working-directory: .
+      run: docker compose up --detach --wait
     - name: Test
+      env:
+        S3_TEST: "1"
       run: npm run test
   macos:
     timeout-minutes: 30

.github/workflows/npm-publish.yml

@@ -1,11 +1,13 @@
 name: NPM Publish
 
 on:
-  release:
-    types: [published]
+  push:
+    tags:
+      - "v*"
 
 jobs:
   node:
+    name: vectordb Typescript
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -38,6 +40,7 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
+    name: vectordb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -63,11 +66,12 @@ jobs:
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: node-native-darwin
+          name: node-native-darwin-${{ matrix.config.arch }}
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
 
   nodejs-macos:
+    name: lancedb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -98,7 +102,7 @@ jobs:
             nodejs/dist/*.node
 
   node-linux:
-    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -110,12 +114,11 @@ jobs:
             runner: ubuntu-latest
           - arch: aarch64
             # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
+            runner: warp-ubuntu-latest-arm64-4x
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      # To avoid OOM errors on ARM, we create a swap file.
       - name: Configure aarch64 build
         if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
@@ -134,12 +137,12 @@ jobs:
       - name: Upload Linux Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: node-native-linux
+          name: node-native-linux-${{ matrix.config.arch }}
           path: |
             node/dist/lancedb-vectordb-linux*.tgz
 
   nodejs-linux:
-    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -190,6 +193,7 @@ jobs:
             !nodejs/dist/*.node
 
   node-windows:
+    name: vectordb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -223,6 +227,7 @@ jobs:
             node/dist/lancedb-vectordb-win32*.tgz
 
   nodejs-windows:
+    name: lancedb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -256,6 +261,7 @@ jobs:
             nodejs/dist/*.node
 
   release:
+    name: vectordb NPM Publish
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
@@ -274,12 +280,28 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
+          # Tag beta as "preview" instead of default "latest". See lancedb 
+          # npm publish step for more info.
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            PUBLISH_ARGS="--tag preview"
+          fi
+
           mv */*.tgz .
           for filename in *.tgz; do
-            npm publish $filename
+            npm publish $PUBLISH_ARGS $filename
           done
+      - name: Notify Slack Action
+        uses: ravsamhq/notify-slack-action@2.3.0
+        if: ${{ always() }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   release-nodejs:
+    name: lancedb NPM Publish
     needs: [nodejs-macos, nodejs-linux, nodejs-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
@@ -316,11 +338,32 @@ jobs:
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        run: npm publish --access public
+        # By default, things are published to the latest tag. This is what is
+        # installed by default if the user does not specify a version. This is
+        # good for stable releases, but for pre-releases, we want to publish to
+        # the "preview" tag so they can install with `npm install lancedb@preview`.
+        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
+        run: |
+          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
+            npm publish --access public --tag preview
+          else
+            npm publish --access public
+          fi
+      - name: Notify Slack Action
+        uses: ravsamhq/notify-slack-action@2.3.0
+        if: ${{ always() }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   update-package-lock:
     needs: [release]
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -331,11 +374,13 @@ jobs:
           lfs: true
       - uses: ./.github/workflows/update_package_lock
         with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
   update-package-lock-nodejs:
     needs: [release-nodejs]
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -346,4 +391,70 @@ jobs:
           lfs: true
       - uses: ./.github/workflows/update_package_lock_nodejs
         with:
-          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Release Notes
+        id: release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: ${{ contains('beta', github.ref) }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.release_notes.outputs.changelog }}

.github/workflows/pypi-publish.yml

@@ -1,18 +1,28 @@
 name: PyPI Publish
 
 on:
-  release:
-    types: [published]
+  push:
+    tags:
+      - 'python-v*'
 
 jobs:
   linux:
+    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     timeout-minutes: 60
     strategy:
       matrix:
-        python-minor-version: ["8"]
-        platform:
-          - x86_64
-          - aarch64
+        config:
+          - platform: x86_64
+            manylinux: "2_17"
+            extra_args: ""
+          - platform: x86_64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+          - platform: aarch64
+            manylinux: "2_24"
+            extra_args: ""
+          # We don't build fp16 kernels for aarch64, because it uses 
+          # cross compilation image, which doesn't have a new enough compiler.
     runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
@@ -22,22 +32,22 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.${{ matrix.python-minor-version }}
+          python-version: 3.8
       - uses: ./.github/workflows/build_linux_wheel
         with:
-          python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip"
-          arm-build: ${{ matrix.platform == 'aarch64' }}
+          python-minor-version: 8
+          args: "--release --strip ${{ matrix.config.extra_args }}"
+          arm-build: ${{ matrix.config.platform == 'aarch64' }}
+          manylinux: ${{ matrix.config.manylinux }}
       - uses: ./.github/workflows/upload_wheel
         with:
-          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          repo: "pypi"
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
   mac:
     timeout-minutes: 60
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
-        python-minor-version: ["8"]
         config:
           - target: x86_64-apple-darwin
             runner: macos-13
@@ -48,7 +58,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref }}
           fetch-depth: 0
           lfs: true
       - name: Set up Python
@@ -57,36 +66,95 @@ jobs:
           python-version: 3.12
       - uses: ./.github/workflows/build_mac_wheel
         with:
-          python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip --target ${{ matrix.config.target }}"
+          python-minor-version: 8
+          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
         with:
-          python-minor-version: ${{ matrix.python-minor-version }}
-          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          repo: "pypi"
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
   windows:
     timeout-minutes: 60
     runs-on: windows-latest
-    strategy:
-      matrix:
-        python-minor-version: ["8"]
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref }}
           fetch-depth: 0
           lfs: true
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.${{ matrix.python-minor-version }}
+          python-version: 3.8
       - uses: ./.github/workflows/build_windows_wheel
         with:
-          python-minor-version: ${{ matrix.python-minor-version }}
+          python-minor-version: 8
           args: "--release --strip"
           vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
       - uses: ./.github/workflows/upload_wheel
         with:
-          python-minor-version: ${{ matrix.python-minor-version }}
-          token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
-          repo: "pypi"
+          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          fury_token: ${{ secrets.FURY_TOKEN }}
+  gh-release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Extract version
+        id: extract_version
+        env:
+          GITHUB_REF: ${{ github.ref }}
+        run: |
+          set -e
+          echo "Extracting tag and version from $GITHUB_REF"
+          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
+            VERSION=${BASH_REMATCH[1]}
+            TAG=python-v$VERSION
+            echo "tag=$TAG" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            echo "Failed to extract version from $GITHUB_REF"
+            exit 1
+          fi
+          echo "Extracted version $VERSION from $GITHUB_REF"
+          if [[ $VERSION =~ beta ]]; then
+            echo "This is a beta release"
+
+            # Get last release (that is not this one)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^python-v \
+              | grep -vF "$TAG" \
+              | python ci/semver_sort.py python-v \
+              | tail -n 1)
+          else
+            echo "This is a stable release"
+            # Get last stable tag (ignore betas)
+            FROM_TAG=$(git tag --sort='version:refname' \
+              | grep ^python-v \
+              | grep -vF "$TAG" \
+              | grep -v beta \
+              | python ci/semver_sort.py python-v \
+              | tail -n 1)
+          fi
+          echo "Found from tag $FROM_TAG"
+          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
+      - name: Create Python Release Notes
+        id: python_release_notes
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: .github/release_notes.json
+          toTag: ${{ steps.extract_version.outputs.tag }}
+          fromTag: ${{ steps.extract_version.outputs.from_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create Python GH release
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: ${{ contains('beta', github.ref) }}
+          tag_name: ${{ steps.extract_version.outputs.tag }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          generate_release_notes: false
+          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
+          body: ${{ steps.python_release_notes.outputs.changelog }}

.github/workflows/python-make-release-commit.yml

@@ -1,56 +0,0 @@
-name: Python - Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Dry run (create the local commit/tags but do not push it)'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v4
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Set git configs for bumpversion
-      shell: bash
-      run: |
-        git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.11"
-    - name: Bump version, create tag and commit
-      working-directory: python
-      run: |
-        pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
-        tags: true
-

.github/workflows/python.yml

@@ -33,11 +33,11 @@ jobs:
           python-version: "3.11"
       - name: Install ruff
         run: |
-          pip install ruff==0.2.2
+          pip install ruff==0.5.4
       - name: Format check
         run: ruff format --check .
       - name: Lint
-        run: ruff .
+        run: ruff check .
   doctest:
     name: "Doctest"
     timeout-minutes: 30
@@ -65,7 +65,7 @@ jobs:
           workspaces: python
       - name: Install
         run: |
-          pip install -e .[tests,dev,embeddings]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
           pip install tantivy
           pip install mlx
       - name: Doctest
@@ -75,7 +75,7 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        python-minor-version: ["8", "11"]
+        python-minor-version: ["9", "11"]
     runs-on: "ubuntu-22.04"
     defaults:
       run:
@@ -99,6 +99,8 @@ jobs:
           workspaces: python
       - uses: ./.github/workflows/build_linux_wheel
       - uses: ./.github/workflows/run_tests
+        with:
+          integration: true
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
@@ -187,7 +189,7 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
-          pip install -e .[tests]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests
-        run: pytest -m "not slow" -x -v --durations=30 python/tests
+        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -5,13 +5,27 @@ inputs:
   python-minor-version:
     required: true
     description: "8 9 10 11 12"
+  integration:
+    required: false
+    description: "Run integration tests"
+    default: "false"
 runs:
   using: "composite"
   steps:
     - name: Install lancedb
       shell: bash
       run: |
-        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
-    - name: pytest
+        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
+    - name: Setup localstack for integration tests
+      if: ${{ inputs.integration == 'true' }}
       shell: bash
+      working-directory: .
+      run: docker compose up --detach --wait
+    - name: pytest (with integration)
+      shell: bash
+      if: ${{ inputs.integration == 'true' }}
       run: pytest -m "not slow" -x -v --durations=30 python/python/tests
+    - name: pytest (no integration tests)
+      shell: bash
+      if: ${{ inputs.integration != 'true' }}
+      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -31,6 +31,10 @@ jobs:
       run:
         shell: bash
         working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -49,11 +53,18 @@ jobs:
       run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
-    runs-on: ubuntu-22.04
+    # To build all features, we need more disk space than is available
+    # on the GitHub-provided runner. This is mostly due to the the
+    # sentence-transformers feature.
+    runs-on: warp-ubuntu-latest-x64-4x
     defaults:
       run:
         shell: bash
         working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -66,6 +77,9 @@ jobs:
       run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
+    - name: Start S3 integration test environment
+      working-directory: .
+      run: docker compose up --detach --wait
     - name: Build
       run: cargo build --all-features
     - name: Run tests
@@ -97,7 +111,8 @@ jobs:
       - name: Build
         run: cargo build --all-features
       - name: Run tests
-        run: cargo test --all-features
+        # Run with everything except the integration tests.
+        run: cargo test --features remote,fp16kernels
   windows:
     runs-on: windows-2022
     steps:
@@ -119,4 +134,3 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
-  

.github/workflows/upload_wheel/action.yml

@@ -2,28 +2,43 @@ name: upload-wheel
 
 description: "Upload wheels to Pypi"
 inputs:
-  os:
-    required: true
-    description: "ubuntu-22.04 or macos-13"
-  repo:
-    required: false
-    description: "pypi or testpypi"
-    default: "pypi"
-  token:
+  pypi_token:
     required: true
     description: "release token for the repo"
+  fury_token:
+    required: true
+    description: "release token for the fury repo"
 
 runs:
   using: "composite"
   steps:
-    - name: Install dependencies
-      shell: bash
-      run: |
-        python -m pip install --upgrade pip
-        pip install twine
-    - name: Publish wheel
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ inputs.token }}
-      shell: bash
-      run: twine upload --repository ${{ inputs.repo }} target/wheels/lancedb-*.whl
+  - name: Install dependencies
+    shell: bash
+    run: |
+      python -m pip install --upgrade pip
+      pip install twine
+  - name: Choose repo
+    shell: bash
+    id: choose_repo
+    run: |
+      if [ ${{ github.ref }} == "*beta*" ]; then
+        echo "repo=fury" >> $GITHUB_OUTPUT
+      else
+        echo "repo=pypi" >> $GITHUB_OUTPUT
+      fi
+  - name: Publish to PyPI
+    shell: bash
+    env:
+      FURY_TOKEN: ${{ inputs.fury_token }}
+      PYPI_TOKEN: ${{ inputs.pypi_token }}
+    run: |
+      if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then
+        WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
+        echo "Uploading $WHEEL to Fury"
+        curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
+      else
+        twine upload --repository ${{ steps.choose_repo.outputs.repo }} \
+          --username __token__ \
+          --password $PYPI_TOKEN \
+          target/wheels/lancedb-*.whl
+      fi

.gitignore

@@ -4,9 +4,10 @@
 **/__pycache__
 .DS_Store
 venv
+.venv
 
 .vscode
-
+.zed
 rust/target
 rust/Cargo.lock
 

.pre-commit-config.yaml

@@ -10,9 +10,12 @@ repos:
     rev: v0.2.2
     hooks:
     - id: ruff
-- repo: https://github.com/pre-commit/mirrors-prettier
-  rev: v3.1.0
+- repo: local
   hooks:
-    - id: prettier
+    - id: local-biome-check
+      name: biome check
+      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
+      language: system
+      types: [text]
       files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*

Cargo.toml

@@ -1,5 +1,11 @@
 [workspace]
-members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"]
+members = [
+    "rust/ffi/node",
+    "rust/lancedb",
+    "nodejs",
+    "python",
+    "java/core/lancedb-jni",
+]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -14,27 +20,30 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.10.5", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.10.5" }
-lance-linalg = { "version" = "=0.10.5" }
-lance-testing = { "version" = "=0.10.5" }
+lance = { "version" = "=0.16.0", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.16.0" }
+lance-linalg = { "version" = "=0.16.0" }
+lance-testing = { "version" = "=0.16.0" }
+lance-datafusion = { "version" = "=0.16.0" }
+lance-encoding = { "version" = "=0.16.0" }
 # Note that this one does not include pyarrow
-arrow = { version = "50.0", optional = false }
-arrow-array = "50.0"
-arrow-data = "50.0"
-arrow-ipc = "50.0"
-arrow-ord = "50.0"
-arrow-schema = "50.0"
-arrow-arith = "50.0"
-arrow-cast = "50.0"
+arrow = { version = "52.2", optional = false }
+arrow-array = "52.2"
+arrow-data = "52.2"
+arrow-ipc = "52.2"
+arrow-ord = "52.2"
+arrow-schema = "52.2"
+arrow-arith = "52.2"
+arrow-cast = "52.2"
 async-trait = "0"
 chrono = "0.4.35"
-half = { "version" = "=2.3.1", default-features = false, features = [
+datafusion-physical-plan = "40.0"
+half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
 ] }
 futures = "0"
 log = "0.4"
-object_store = "0.9.0"
+object_store = "0.10.1"
 pin-project = "1.0.7"
 snafu = "0.7.4"
 url = "2"

README.md

@@ -1,14 +1,14 @@
 <div align="center">
 <p align="center">
 
-<img width="275" alt="LanceDB Logo" src="https://user-images.githubusercontent.com/917119/226205734-6063d87a-1ecc-45fe-85be-1dea6383a3d8.png">
+<img width="275" alt="LanceDB Logo" src="https://github.com/lancedb/lancedb/assets/5846846/37d7c7ad-c2fd-4f56-9f16-fffb0d17c73a">
 
-**Developer-friendly, serverless vector database for AI applications**
+**Developer-friendly, database for multimodal AI**
 
 <a href='https://github.com/lancedb/vectordb-recipes/tree/main' target="_blank"><img alt='LanceDB' src='https://img.shields.io/badge/VectorDB_Recipes-100000?style=for-the-badge&logo=LanceDB&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
 <a href='https://lancedb.github.io/lancedb/' target="_blank"><img alt='lancdb' src='https://img.shields.io/badge/DOCS-100000?style=for-the-badge&logo=lancdb&logoColor=white&labelColor=645cfb&color=645cfb'/></a>
-[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://blog.lancedb.com/) 
-[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd) 
+[![Blog](https://img.shields.io/badge/Blog-12100E?style=for-the-badge&logoColor=white)](https://blog.lancedb.com/)
+[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/zMM32dvNtd)
 [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/lancedb)
 
 </p>
@@ -20,7 +20,7 @@
 
 <hr />
 
-LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
+LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings.
 
 The key features of LanceDB include:
 
@@ -36,7 +36,7 @@ The key features of LanceDB include:
 
 * GPU support in building vector index(*).
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/docs/integrations/vectorstores/lancedb/), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -44,26 +44,24 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
 **Javascript**
 ```shell
-npm install vectordb
+npm install @lancedb/lancedb
 ```
 
 ```javascript
-const lancedb = require('vectordb');
-const db = await lancedb.connect('data/sample-lancedb');
+import * as lancedb from "@lancedb/lancedb";
 
-const table = await db.createTable({
-  name: 'vectors',
-  data:  [
-    { id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
-    { id: 2, vector: [1.1, 1.2], item: "bar", price: 50 }
-  ]
-})
+const db = await lancedb.connect("data/sample-lancedb");
+const table = await db.createTable("vectors", [
+	{ id: 1, vector: [0.1, 0.2], item: "foo", price: 10 },
+	{ id: 2, vector: [1.1, 1.2], item: "bar", price: 50 },
+], {mode: 'overwrite'});
 
-const query = table.search([0.1, 0.3]).limit(2);
-const results = await query.execute();
+
+const query = table.vectorSearch([0.1, 0.3]).limit(2);
+const results = await query.toArray();
 
 // You can also search for rows by specific criteria without involving a vector search.
-const rowsByCriteria = await table.search(undefined).where("price >= 10").execute();
+const rowsByCriteria = await table.query().where("price >= 10").toArray();
 ```
 
 **Python**
@@ -83,5 +81,5 @@ result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
+* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
 * 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -18,4 +18,4 @@ docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
     lancedb-node-manylinux \
-    bash ci/manylinux_node/build.sh $ARCH
+    bash ci/manylinux_node/build_vectordb.sh $ARCH

ci/build_linux_artifacts_nodejs.sh

@@ -4,9 +4,9 @@ ARCH=${1:-x86_64}
 
 # We pass down the current user so that when we later mount the local files
 # into the container, the files are accessible by the current user.
-pushd ci/manylinux_nodejs
+pushd ci/manylinux_node
 docker build \
-    -t lancedb-nodejs-manylinux \
+    -t lancedb-node-manylinux-$ARCH \
     --build-arg="ARCH=$ARCH" \
     --build-arg="DOCKER_USER=$(id -u)" \
     --progress=plain \
@@ -17,5 +17,5 @@ popd
 docker run \
     -v $(pwd):/io -w /io \
     --memory-swap=-1 \
-    lancedb-nodejs-manylinux \
-    bash ci/manylinux_nodejs/build.sh $ARCH
+    lancedb-node-manylinux-$ARCH \
+    bash ci/manylinux_node/build_lancedb.sh $ARCH

ci/bump_version.sh

@@ -0,0 +1,51 @@
+set -e
+
+RELEASE_TYPE=${1:-"stable"}
+BUMP_MINOR=${2:-false}
+TAG_PREFIX=${3:-"v"} # Such as "python-v"
+HEAD_SHA=${4:-$(git rev-parse HEAD)}
+
+readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
+echo "Found previous tag $PREV_TAG"
+
+# Initially, we don't want to tag if we are doing stable, because we will bump
+# again later. See comment at end for why.
+if [[ "$RELEASE_TYPE" == 'stable' ]]; then 
+  BUMP_ARGS="--no-tag"
+fi
+
+# If last is stable and not bumping minor
+if [[ $PREV_TAG != *beta* ]]; then
+    if [[ "$BUMP_MINOR" != "false" ]]; then
+      # X.Y.Z -> X.(Y+1).0-beta.0
+      bump-my-version bump -vv $BUMP_ARGS minor
+    else
+      # X.Y.Z -> X.Y.(Z+1)-beta.0
+      bump-my-version bump -vv $BUMP_ARGS patch
+    fi
+else
+  if [[ "$BUMP_MINOR" != "false" ]]; then
+    # X.Y.Z-beta.N -> X.(Y+1).0-beta.0
+    bump-my-version bump -vv $BUMP_ARGS minor
+  else
+    # X.Y.Z-beta.N -> X.Y.Z-beta.(N+1)
+    bump-my-version bump -vv $BUMP_ARGS pre_n
+  fi
+fi
+
+# The above bump will always bump to a pre-release version. If we are releasing
+# a stable version, bump the pre-release level ("pre_l") to make it stable.
+if [[ $RELEASE_TYPE == 'stable' ]]; then
+  # X.Y.Z-beta.N -> X.Y.Z
+  bump-my-version bump -vv pre_l
+fi
+
+# Validate that we have incremented version appropriately for breaking changes
+NEW_TAG=$(git describe --tags --exact-match HEAD)
+NEW_VERSION=$(echo $NEW_TAG | sed "s/^$TAG_PREFIX//")
+LAST_STABLE_RELEASE=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | grep -v beta | grep -vF "$NEW_TAG" | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1)
+LAST_STABLE_VERSION=$(echo $LAST_STABLE_RELEASE | sed "s/^$TAG_PREFIX//")
+
+python $SELF_DIR/check_breaking_changes.py $LAST_STABLE_RELEASE $HEAD_SHA $LAST_STABLE_VERSION $NEW_VERSION

ci/check_breaking_changes.py

@@ -0,0 +1,35 @@
+"""
+Check whether there are any breaking changes in the PRs between the base and head commits.
+If there are, assert that we have incremented the minor version.
+"""
+import argparse
+import os
+from packaging.version import parse
+
+from github import Github
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base")
+    parser.add_argument("head")
+    parser.add_argument("last_stable_version")
+    parser.add_argument("current_version")
+    args = parser.parse_args()
+
+    repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"])
+    commits = repo.compare(args.base, args.head).commits
+    prs = (pr for commit in commits for pr in commit.get_pulls())
+
+    for pr in prs:
+        if any(label.name == "breaking-change" for label in pr.labels):
+            print(f"Breaking change in PR: {pr.html_url}")
+            break
+    else:
+        print("No breaking changes found.")
+        exit(0)
+    
+    last_stable_version = parse(args.last_stable_version)
+    current_version = parse(args.current_version)
+    if current_version.minor <= last_stable_version.minor:
+        print("Minor version is not greater than the last stable version.")
+        exit(1)

ci/manylinux_node/Dockerfile

@@ -4,7 +4,7 @@
 # range of linux distributions.
 ARG ARCH=x86_64
 
-FROM quay.io/pypa/manylinux2014_${ARCH}
+FROM quay.io/pypa/manylinux_2_28_${ARCH}
 
 ARG ARCH=x86_64
 ARG DOCKER_USER=default_user
@@ -18,8 +18,8 @@ COPY install_protobuf.sh install_protobuf.sh
 RUN ./install_protobuf.sh ${ARCH}
 
 ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+# Create a group and user, but only if it doesn't exist
+RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user
 
 # We switch to the user to install Rust and Node, since those like to be 
 # installed at the user level.

ci/manylinux_node/install_openssl.sh

@@ -6,7 +6,7 @@
 # /usr/bin/ld: failed to set dynamic section sizes: Bad value
 set -e
 
-git clone -b OpenSSL_1_1_1u \
+git clone -b OpenSSL_1_1_1v \
     --single-branch \
     https://github.com/openssl/openssl.git
 

ci/manylinux_node/prepare_manylinux_node.sh

@@ -8,7 +8,7 @@ install_node() {
 
     source "$HOME"/.bashrc
 
-    nvm install --no-progress 16
+    nvm install --no-progress 18
 }
 
 install_rust() {

ci/manylinux_nodejs/Dockerfile

@@ -1,31 +0,0 @@
-# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
-# This container allows building the node modules native libraries in an
-# environment with a very old glibc, so that we are compatible with a wide
-# range of linux distributions.
-ARG ARCH=x86_64
-
-FROM quay.io/pypa/manylinux2014_${ARCH}
-
-ARG ARCH=x86_64
-ARG DOCKER_USER=default_user
-
-# Install static openssl
-COPY install_openssl.sh install_openssl.sh
-RUN ./install_openssl.sh ${ARCH} > /dev/null
-
-# Protobuf is also installed as root.
-COPY install_protobuf.sh install_protobuf.sh
-RUN ./install_protobuf.sh ${ARCH}
-
-ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
-
-# We switch to the user to install Rust and Node, since those like to be
-# installed at the user level.
-USER ${DOCKER_USER}
-
-COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
-RUN cp /prepare_manylinux_node.sh $HOME/ && \
-    cd $HOME && \
-    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_nodejs/install_openssl.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Builds openssl from source so we can statically link to it
-
-# this is to avoid the error we get with the system installation:
-# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
-# /usr/bin/ld: failed to set dynamic section sizes: Bad value
-set -e
-
-git clone -b OpenSSL_1_1_1u \
-    --single-branch \
-    https://github.com/openssl/openssl.git
-
-pushd openssl
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=linux-x86_64
-else
-    # gnu target
-    ARCH=linux-aarch64
-fi
-
-./Configure no-shared $ARCH
-
-make
-
-make install

ci/manylinux_nodejs/install_protobuf.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-# Installs protobuf compiler. Should be run as root.
-set -e
-
-if [[ $1 == x86_64* ]]; then
-    ARCH=x86_64
-else
-    # gnu target
-    ARCH=aarch_64
-fi
-
-PB_REL=https://github.com/protocolbuffers/protobuf/releases
-PB_VERSION=23.1
-curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_nodejs/prepare_manylinux_node.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-install_node() {
-    echo "Installing node..."
-
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-
-    source "$HOME"/.bashrc
-
-    nvm install --no-progress 16
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-install_node
-install_rust

ci/semver_sort.py

@@ -0,0 +1,35 @@
+"""
+Takes a list of semver strings and sorts them in ascending order.
+"""
+
+import sys
+from packaging.version import parse, InvalidVersion
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("prefix", default="v")
+    args = parser.parse_args()
+
+    # Read the input from stdin
+    lines = sys.stdin.readlines()
+
+    # Parse the versions
+    versions = []
+    for line in lines:
+        line = line.strip()
+        try:
+            version_str = line.removeprefix(args.prefix)
+            version = parse(version_str)
+        except InvalidVersion:
+            # There are old tags that don't follow the semver format
+            print(f"Invalid version: {line}", file=sys.stderr)
+            continue
+        versions.append((line, version))
+
+    # Sort the versions
+    versions.sort(key=lambda x: x[1])
+
+    # Print the sorted versions as original strings
+    for line, _ in versions:
+        print(line)

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:0.14
+    image: localstack/localstack:3.3
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb
+      - SERVICES=s3,dynamodb,kms
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
+      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
       interval: 5s
       retries: 3
       start_period: 10s

docs/mkdocs.yml

@@ -57,16 +57,8 @@ plugins:
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
   - mkdocs-jupyter
-  - ultralytics:
-      verbose: True
-      enabled: True
-      default_image: "assets/lancedb_and_lance.png" # Default image for all pages
-      add_image: True # Automatically add meta image
-      add_keywords: True # Add page keywords in the header tag
-      add_share_buttons: True # Add social share buttons
-      add_authors: False # Display page authors
-      add_desc: False
-      add_dates: False
+  - render_swagger:
+      allow_arbitrary_locations : true
 
 markdown_extensions:
   - admonition
@@ -85,7 +77,7 @@ markdown_extensions:
       alternate_style: true
   - md_in_html
   - attr_list
-  
+
 nav:
   - Home:
       - LanceDB: index.md
@@ -108,14 +100,21 @@ nav:
               - Quickstart: reranking/index.md
               - Cohere Reranker: reranking/cohere.md
               - Linear Combination Reranker: reranking/linear_combination.md
+              - Reciprocal Rank Fusion Reranker: reranking/rrf.md
               - Cross Encoder Reranker: reranking/cross_encoder.md
               - ColBERT Reranker: reranking/colbert.md
+              - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
+              - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
           - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
-          - Sync -> Async Migration Guide: migration.md
+          - Migration Guide: migration.md
+          - Tuning retrieval performance:
+              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+              - Reranking: guides/tuning_retrievers/2_reranking.md
+              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
           - Overview: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
@@ -128,9 +127,13 @@ nav:
           - Pandas and PyArrow: python/pandas_and_pyarrow.md
           - Polars: python/polars_arrow.md
           - DuckDB: python/duckdb.md
-          - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-          - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-          - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+          - LangChain:
+            - LangChain 🔗: integrations/langchain.md
+            - LangChain demo: notebooks/langchain_demo.ipynb
+            - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+          - LlamaIndex 🦙:
+            - LlamaIndex docs: integrations/llamaIndex.md
+            - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
           - Pydantic: python/pydantic.md
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
@@ -138,12 +141,15 @@ nav:
           - Overview: examples/index.md
           - 🐍 Python:
               - Overview: examples/examples_python.md
-              - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-              - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-              - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-              - Example - Calculate CLIP Embeddings with Roboflow Inference: examples/image_embeddings_roboflow.md
-              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+              - Build From Scratch: examples/python_examples/build_from_scratch.md
+              - Multimodal: examples/python_examples/multimodal.md
+              - Rag: examples/python_examples/rag.md
+              - Miscellaneous: 
+                  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+                  - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+                  - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+                  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+                  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
           - 👾 JavaScript:
               - Overview: examples/examples_js.md
               - Serverless Website Chatbot: examples/serverless_website_chatbot.md
@@ -151,17 +157,18 @@ nav:
               - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
           - 🦀 Rust:
               - Overview: examples/examples_rust.md
-      - 🔧 CLI & Config: cli_config.md
       - 💭 FAQs: faq.md
       - ⚙️ API reference:
           - 🐍 Python: python/python.md
-          - 👾 JavaScript: javascript/modules.md
+          - 👾 JavaScript (vectordb): javascript/modules.md
+          - 👾 JavaScript (lancedb): js/globals.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
       - ☁️ LanceDB Cloud:
           - Overview: cloud/index.md
           - API reference:
               - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/saas-modules.md
+              - 👾 JavaScript: javascript/modules.md
+              - REST API: cloud/rest.md
 
   - Quick start: basic.md
   - Concepts:
@@ -182,14 +189,21 @@ nav:
           - Quickstart: reranking/index.md
           - Cohere Reranker: reranking/cohere.md
           - Linear Combination Reranker: reranking/linear_combination.md
+          - Reciprocal Rank Fusion Reranker: reranking/rrf.md
           - Cross Encoder Reranker: reranking/cross_encoder.md
           - ColBERT Reranker: reranking/colbert.md
+          - Jina Reranker: reranking/jina.md
           - OpenAI Reranker: reranking/openai.md
           - Building Custom Rerankers: reranking/custom_reranker.md
+          - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
       - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
-      - Sync -> Async Migration Guide: migration.md
+      - Migration Guide: migration.md
+      - Tuning retrieval performance:
+          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+          - Reranking: guides/tuning_retrievers/2_reranking.md
+          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
       - Overview: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
@@ -202,32 +216,44 @@ nav:
       - Pandas and PyArrow: python/pandas_and_pyarrow.md
       - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
-      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
-      - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+      - LangChain 🦜️🔗↗: integrations/langchain.md
+      - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
+      - LlamaIndex 🦙↗: integrations/llamaIndex.md
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
   - Examples:
       - examples/index.md
-      - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
-      - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
-      - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
-      - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
-      - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
-      - YouTube Transcript Search (JS): examples/youtube_transcript_bot_with_nodejs.md
-      - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
-      - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+      - 🐍 Python:
+          - Overview: examples/examples_python.md
+          - Build From Scratch: examples/python_examples/build_from_scratch.md
+          - Multimodal: examples/python_examples/multimodal.md
+          - Rag: examples/python_examples/rag.md
+          - Miscellaneous: 
+              - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+              - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+              - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+              - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+              - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+      - 👾 JavaScript:
+          - Overview: examples/examples_js.md
+          - Serverless Website Chatbot: examples/serverless_website_chatbot.md
+          - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+          - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+      - 🦀 Rust:
+          - Overview: examples/examples_rust.md
   - API reference:
       - Overview: api_reference.md
       - Python: python/python.md
-      - Javascript: javascript/modules.md
+      - Javascript (vectordb): javascript/modules.md
+      - Javascript (lancedb): js/globals.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
   - LanceDB Cloud:
       - Overview: cloud/index.md
       - API reference:
           - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/saas-modules.md
+          - 👾 JavaScript: javascript/modules.md
+          - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css
@@ -239,4 +265,11 @@ extra_javascript:
 extra:
   analytics:
     provider: google
-    property: G-B7NFM40W74
+    property: G-B7NFM40W74
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/lancedb/lancedb
+    - icon: fontawesome/brands/x-twitter
+      link: https://twitter.com/lancedb
+    - icon: fontawesome/brands/linkedin
+      link: https://www.linkedin.com/company/lancedb

docs/openapi.yml

@@ -0,0 +1,487 @@
+openapi: 3.1.0
+info:
+  version: 1.0.0
+  title: LanceDB Cloud API
+  description: |
+    LanceDB Cloud API is a RESTful API that allows users to access and modify data stored in LanceDB Cloud.
+    Table actions are considered temporary resource creations and all use POST method.
+  contact:
+    name: LanceDB support
+    url: https://lancedb.com
+    email: contact@lancedb.com
+
+servers:
+  - url: https://{db}.{region}.api.lancedb.com
+    description: LanceDB Cloud REST endpoint.
+    variables:
+      db:
+        default: ""
+        description: the name of DB
+      region:
+        default: "us-east-1"
+        description: the service region of the DB
+
+security:
+  - key_auth: []
+
+components:
+  securitySchemes:
+    key_auth:
+      name: x-api-key
+      type: apiKey
+      in: header
+  parameters:
+    table_name:
+      name: name
+      in: path
+      description: name of the table
+      required: true
+      schema:
+        type: string
+  responses:
+    invalid_request:
+      description: Invalid request
+      content:
+        text/plain:
+          schema:
+            type: string
+    not_found:
+      description: Not found
+      content:
+        text/plain:
+          schema:
+            type: string
+    unauthorized:
+      description: Unauthorized
+      content:
+        text/plain:
+          schema:
+            type: string
+  requestBodies:
+    arrow_stream_buffer:
+      description: Arrow IPC stream buffer
+      required: true
+      content:
+        application/vnd.apache.arrow.stream:
+          schema:
+            type: string
+            format: binary
+
+paths:
+  /v1/table/:
+    get:
+      description: List tables, optionally, with pagination.
+      tags:
+        - Tables
+      summary: List Tables
+      operationId: listTables
+      parameters:
+        - name: limit
+          in: query
+          description: Limits the number of items to return.
+          schema:
+            type: integer
+        - name: page_token
+          in: query
+          description: Specifies the starting position of the next query
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Successfully returned a list of tables in the DB
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tables:
+                    type: array
+                    items:
+                      type: string
+                  page_token:
+                    type: string
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/create/:
+    post:
+      description: Create a new table
+      summary: Create a new table
+      operationId: createTable
+      tags:
+        - Tables
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Table successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/query/:
+    post:
+      description: Vector Query
+      url: https://{db-uri}.{aws-region}.api.lancedb.com/v1/table/{name}/query/
+      tags:
+        - Data
+      summary: Vector Query
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                vector:
+                  type: FixedSizeList
+                  description: |
+                    The targetted vector to search for. Required.
+                vector_column:
+                  type: string
+                  description: |
+                    The column to query, it can be inferred from the schema if there is only one vector column.
+                prefilter:
+                  type: boolean
+                  description: |
+                    Whether to prefilter the data. Optional.
+                k:
+                  type: integer
+                  description: |
+                    The number of search results to return. Default is 10.
+                distance_type:
+                  type: string
+                  description: |
+                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                bypass_vector_index:
+                  type: boolean
+                  description: |
+                    Whether to bypass vector index. Optional.
+                filter:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to query. Optional.
+                columns:
+                  type: array
+                  items:
+                    type: string
+                  description: |
+                    The columns to return. Optional.
+                nprobe:
+                  type: integer
+                  description: |
+                    The number of probes to use for search. Optional.
+                refine_factor:
+                  type: integer
+                  description: |
+                    The refine factor to use for search. Optional.
+                  default: null
+                fast_search:
+                  type: boolean
+                  description: |
+                    Whether to use fast search. Optional.
+                  default: false
+              required:
+                - vector
+
+      responses:
+        "200":
+          description: top k results if query is successfully executed
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  results:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        id:
+                          type: integer
+                        selected_col_1_to_return:
+                          type: col_1_type
+                        selected_col_n_to_return:
+                          type: col_n_type
+                        _distance:
+                          type: float
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/insert/:
+    post:
+      description: Insert new data to the Table.
+      tags:
+        - Data
+      operationId: insertData
+      summary: Insert new data.
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/merge_insert/:
+    post:
+      description: Create a "merge insert" operation
+        This operation can add rows, update rows, and remove rows all in a single
+        transaction. See python method `lancedb.table.Table.merge_insert` for examples.
+      tags:
+        - Data
+      summary: Merge Insert
+      operationId: mergeInsert
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+        - name: on
+          in: query
+          description: |
+            The column to use as the primary key for the merge operation.
+          required: true
+          schema:
+            type: string
+        - name: when_matched_update_all
+          in: query
+          description: |
+            Rows that exist in both the source table (new data) and
+            the target table (old data) will be updated, replacing
+            the old row with the corresponding matching row.
+          required: false
+          schema:
+            type: boolean
+        - name: when_matched_update_all_filt
+          in: query
+          description: |
+            If present then only rows that satisfy the filter expression will
+            be updated
+          required: false
+          schema:
+            type: string
+        - name: when_not_matched_insert_all
+          in: query
+          description: |
+            Rows that exist only in the source table (new data) will be
+            inserted into the target table (old data).
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete
+          in: query
+          description: |
+            Rows that exist only in the target table (old data) will be
+            deleted. An optional condition (`when_not_matched_by_source_delete_filt`)
+            can be provided to limit what data is deleted.
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete_filt
+          in: query
+          description: |
+            The filter expression that specifies the rows to delete.
+          required: false
+          schema:
+            type: string
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Merge Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/delete/:
+    post:
+      description: Delete rows from a table.
+      tags:
+        - Data
+      summary: Delete rows from a table
+      operationId: deleteData
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                predicate:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to delete.
+      responses:
+        "200":
+          description: Delete successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+  /v1/table/{name}/drop/:
+    post:
+      description: Drop a table
+      tags:
+        - Tables
+      summary: Drop a table
+      operationId: dropTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Drop successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+
+  /v1/table/{name}/describe/:
+    post:
+      description: Describe a table and return Table Information.
+      tags:
+        - Tables
+      summary: Describe a table
+      operationId: describeTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Table information
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  table:
+                    type: string
+                  version:
+                    type: integer
+                  schema:
+                    type: string
+                  stats:
+                    type: object
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/index/list/:
+    post:
+      description: List indexes of a table
+      tags:
+        - Tables
+      summary: List indexes of a table
+      operationId: listIndexes
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Available list of indexes on the table.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  indexes:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        columns:
+                          type: array
+                          items:
+                            type: string
+                        index_name:
+                          type: string
+                        index_uuid:
+                          type: string
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_index/:
+    post:
+      description: Create vector index on a Table
+      tags:
+        - Tables
+      summary: Create vector index on a Table
+      operationId: createIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                metric_type:
+                  type: string
+                  nullable: false
+                  description: |
+                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                index_type:
+                  type: string
+      responses:
+        "200":
+          description: Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_scalar_index/:
+    post:
+      description: Create a scalar index on a table
+      tags:
+        - Tables
+      summary: Create a scalar index on a table
+      operationId: createScalarIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                index_type:
+                  type: string
+                  required: false
+      responses:
+        "200":
+          description: Scalar Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"

docs/requirements.txt

@@ -2,5 +2,5 @@ mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
+mkdocs-render-swagger-plugin
 pydantic
-mkdocs-ultralytics-plugin==0.0.44

docs/src/ann_indexes.md

@@ -38,13 +38,27 @@ Lance supports `IVF_PQ` index type by default.
     tbl.create_index(num_partitions=256, num_sub_vectors=96)
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
-    --8<--- "docs/src/ann_indexes.ts:import"
+    === "@lancedb/lancedb"
 
-    --8<-- "docs/src/ann_indexes.ts:ingest"
-    ```
+        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
+
+        ```typescript
+        --8<--- "nodejs/examples/ann_indexes.ts:import"
+
+        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        ```
+
+    === "vectordb (deprecated)"
+
+        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
+
+        ```typescript
+        --8<--- "docs/src/ann_indexes.ts:import"
+
+        --8<-- "docs/src/ann_indexes.ts:ingest"
+        ```
 
 === "Rust"
 
@@ -91,27 +105,27 @@ You can specify the GPU device to train IVF partitions via
 
 === "Linux"
 
-     <!-- skip-test -->
-     ``` { .python .copy }
-     # Create index using CUDA on Nvidia GPUs.
-     tbl.create_index(
-          num_partitions=256,
-          num_sub_vectors=96,
-          accelerator="cuda"
-     )
-     ```
+    <!-- skip-test -->
+    ``` { .python .copy }
+    # Create index using CUDA on Nvidia GPUs.
+    tbl.create_index(
+        num_partitions=256,
+        num_sub_vectors=96,
+        accelerator="cuda"
+    )
+    ```
 
 === "MacOS"
 
-     <!-- skip-test -->
-     ```python
-     # Create index using MPS on Apple Silicon.
-     tbl.create_index(
-          num_partitions=256,
-          num_sub_vectors=96,
-          accelerator="mps"
-     )
-     ```
+    <!-- skip-test -->
+    ```python
+    # Create index using MPS on Apple Silicon.
+    tbl.create_index(
+        num_partitions=256,
+        num_sub_vectors=96,
+        accelerator="mps"
+    )
+    ```
 
 Troubleshooting:
 
@@ -150,11 +164,19 @@ There are a couple of parameters that can be used to fine-tune the search:
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
-    --8<-- "docs/src/ann_indexes.ts:search1"
-    ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search1"
+        ```
 
 === "Rust"
 
@@ -172,15 +194,23 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
 
-     ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
-     ```
+    ```python
+    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```javascript
-     --8<-- "docs/src/ann_indexes.ts:search2"
-     ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```javascript
+        --8<-- "docs/src/ann_indexes.ts:search2"
+        ```
 
 ### Projections (select clause)
 
@@ -188,23 +218,31 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-     ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-     ```
+    ```python
+    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+    ```
 
 
-     ```text
-                                                   vector _distance
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
-     ```
+    ```text
+                                                vector _distance
+    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+    ...
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```typescript
-     --8<-- "docs/src/ann_indexes.ts:search3"
-     ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search3"
+        ```
 
 ## FAQ
 

docs/src/api_reference.md

@@ -3,5 +3,6 @@
 The API reference for the LanceDB client SDKs are available at the following locations:
 
 - [Python](python/python.md)
-- [JavaScript](javascript/modules.md)
+- [JavaScript (legacy vectordb package)](javascript/modules.md)
+- [JavaScript (newer @lancedb/lancedb package)](js/globals.md)
 - [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/assets/colab.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="117" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="117" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h30v20H0z"/><path fill="#007ec6" d="M30 0h87v20H30z"/><path fill="url(#b)" d="M0 0h117v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"><svg x="4px" y="0px" width="22px" height="20px" viewBox="-2 0 28 24" style="background-color: #fff;border-radius: 1px;"><path style="fill:#e8710a;" d="M1.977,16.77c-2.667-2.277-2.605-7.079,0-9.357C2.919,8.057,3.522,9.075,4.49,9.691c-1.152,1.6-1.146,3.201-0.004,4.803C3.522,15.111,2.918,16.126,1.977,16.77z"/><path style="fill:#f9ab00;" d="M12.257,17.114c-1.767-1.633-2.485-3.658-2.118-6.02c0.451-2.91,2.139-4.893,4.946-5.678c2.565-0.718,4.964-0.217,6.878,1.819c-0.884,0.743-1.707,1.547-2.434,2.446C18.488,8.827,17.319,8.435,16,8.856c-2.404,0.767-3.046,3.241-1.494,5.644c-0.241,0.275-0.493,0.541-0.721,0.826C13.295,15.939,12.511,16.3,12.257,17.114z"/><path style="fill:#e8710a;" d="M19.529,9.682c0.727-0.899,1.55-1.703,2.434-2.446c2.703,2.783,2.701,7.031-0.005,9.764c-2.648,2.674-6.936,2.725-9.701,0.115c0.254-0.814,1.038-1.175,1.528-1.788c0.228-0.285,0.48-0.552,0.721-0.826c1.053,0.916,2.254,1.268,3.6,0.83C20.502,14.551,21.151,11.927,19.529,9.682z"/><path style="fill:#f9ab00;" d="M4.49,9.691C3.522,9.075,2.919,8.057,1.977,7.413c2.209-2.398,5.721-2.942,8.476-1.355c0.555,0.32,0.719,0.606,0.285,1.128c-0.157,0.188-0.258,0.422-0.391,0.631c-0.299,0.47-0.509,1.067-0.929,1.371C8.933,9.539,8.523,8.847,8.021,8.746C6.673,8.475,5.509,8.787,4.49,9.691z"/><path style="fill:#f9ab00;" d="M1.977,16.77c0.941-0.644,1.545-1.659,2.509-2.277c1.373,1.152,2.85,1.433,4.45,0.499c0.332-0.194,0.503-0.088,0.673,0.19c0.386,0.635,0.753,1.285,1.181,1.89c0.34,0.48,0.222,0.715-0.253,1.006C7.84,19.73,4.205,19.188,1.977,16.77z"/></svg><text x="245" y="140" transform="scale(.1)" textLength="30"> </text><text x="725" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="770">Open in Colab</text><text x="725" y="140" transform="scale(.1)" textLength="770">Open in Colab</text></g> </svg>

docs/src/assets/ghost.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="88.25" height="28" role="img" aria-label="GHOST"><title>GHOST</title><g shape-rendering="crispEdges"><rect width="88.25" height="28" fill="#000"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZjdkZjFlIiByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+R2hvc3Q8L3RpdGxlPjxwYXRoIGQ9Ik0xMiAwQzUuMzczIDAgMCA1LjM3MyAwIDEyczUuMzczIDEyIDEyIDEyIDEyLTUuMzczIDEyLTEyUzE4LjYyNyAwIDEyIDB6bS4yNTYgMi4zMTNjMi40Ny4wMDUgNS4xMTYgMi4wMDggNS44OTggMi45NjJsLjI0NC4zYzEuNjQgMS45OTQgMy41NjkgNC4zNCAzLjU2OSA2Ljk2NiAwIDMuNzE5LTIuOTggNS44MDgtNi4xNTggNy41MDgtMS40MzMuNzY2LTIuOTggMS41MDgtNC43NDggMS41MDgtNC41NDMgMC04LjM2Ni0zLjU2OS04LjM2Ni04LjExMiAwLS43MDYuMTctMS40MjUuMzQyLTIuMTUuMTIyLS41MTUuMjQ0LTEuMDMzLjMwNy0xLjU0OS41NDgtNC41MzkgMi45NjctNi43OTUgOC40MjItNy40MDhhNC4yOSA0LjI5IDAgMDEuNDktLjAyNloiLz48L3N2Zz4="/><text transform="scale(.1)" x="541.25" y="175" textLength="442.5" fill="#fff" font-weight="bold">GHOST</text></g></svg>

docs/src/assets/github.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="95.5" height="28" role="img" aria-label="GITHUB"><title>GITHUB</title><g shape-rendering="crispEdges"><rect width="95.5" height="28" fill="#121011"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSJ3aGl0ZSIgcm9sZT0iaW1nIiB2aWV3Qm94PSIwIDAgMjQgMjQiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+PHRpdGxlPkdpdEh1YjwvdGl0bGU+PHBhdGggZD0iTTEyIC4yOTdjLTYuNjMgMC0xMiA1LjM3My0xMiAxMiAwIDUuMzAzIDMuNDM4IDkuOCA4LjIwNSAxMS4zODUuNi4xMTMuODItLjI1OC44Mi0uNTc3IDAtLjI4NS0uMDEtMS4wNC0uMDE1LTIuMDQtMy4zMzguNzI0LTQuMDQyLTEuNjEtNC4wNDItMS42MUM0LjQyMiAxOC4wNyAzLjYzMyAxNy43IDMuNjMzIDE3LjdjLTEuMDg3LS43NDQuMDg0LS43MjkuMDg0LS43MjkgMS4yMDUuMDg0IDEuODM4IDEuMjM2IDEuODM4IDEuMjM2IDEuMDcgMS44MzUgMi44MDkgMS4zMDUgMy40OTUuOTk4LjEwOC0uNzc2LjQxNy0xLjMwNS43Ni0xLjYwNS0yLjY2NS0uMy01LjQ2Ni0xLjMzMi01LjQ2Ni01LjkzIDAtMS4zMS40NjUtMi4zOCAxLjIzNS0zLjIyLS4xMzUtLjMwMy0uNTQtMS41MjMuMTA1LTMuMTc2IDAgMCAxLjAwNS0uMzIyIDMuMyAxLjIzLjk2LS4yNjcgMS45OC0uMzk5IDMtLjQwNSAxLjAyLjAwNiAyLjA0LjEzOCAzIC40MDUgMi4yOC0xLjU1MiAzLjI4NS0xLjIzIDMuMjg1LTEuMjMuNjQ1IDEuNjUzLjI0IDIuODczLjEyIDMuMTc2Ljc2NS44NCAxLjIzIDEuOTEgMS4yMyAzLjIyIDAgNC42MS0yLjgwNSA1LjYyNS01LjQ3NSA1LjkyLjQyLjM2LjgxIDEuMDk2LjgxIDIuMjIgMCAxLjYwNi0uMDE1IDIuODk2LS4wMTUgMy4yODYgMCAuMzE1LjIxLjY5LjgyNS41N0MyMC41NjUgMjIuMDkyIDI0IDE3LjU5MiAyNCAxMi4yOTdjMC02LjYyNy01LjM3My0xMi0xMi0xMiIvPjwvc3ZnPg=="/><text transform="scale(.1)" x="577.5" y="175" textLength="515" fill="#fff" font-weight="bold">GITHUB</text></g></svg>

docs/src/assets/python.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="97.5" height="28" role="img" aria-label="PYTHON"><title>PYTHON</title><g shape-rendering="crispEdges"><rect width="97.5" height="28" fill="#3670a0"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="100"><image x="9" y="7" width="14" height="14" xlink:href="data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZmZkZDU0IiByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+UHl0aG9uPC90aXRsZT48cGF0aCBkPSJNMTQuMjUuMThsLjkuMi43My4yNi41OS4zLjQ1LjMyLjM0LjM0LjI1LjM0LjE2LjMzLjEuMy4wNC4yNi4wMi4yLS4wMS4xM1Y4LjVsLS4wNS42My0uMTMuNTUtLjIxLjQ2LS4yNi4zOC0uMy4zMS0uMzMuMjUtLjM1LjE5LS4zNS4xNC0uMzMuMS0uMy4wNy0uMjYuMDQtLjIxLjAySDguNzdsLS42OS4wNS0uNTkuMTQtLjUuMjItLjQxLjI3LS4zMy4zMi0uMjcuMzUtLjIuMzYtLjE1LjM3LS4xLjM1LS4wNy4zMi0uMDQuMjctLjAyLjIxdjMuMDZIMy4xN2wtLjIxLS4wMy0uMjgtLjA3LS4zMi0uMTItLjM1LS4xOC0uMzYtLjI2LS4zNi0uMzYtLjM1LS40Ni0uMzItLjU5LS4yOC0uNzMtLjIxLS44OC0uMTQtMS4wNS0uMDUtMS4yMy4wNi0xLjIyLjE2LTEuMDQuMjQtLjg3LjMyLS43MS4zNi0uNTcuNC0uNDQuNDItLjMzLjQyLS4yNC40LS4xNi4zNi0uMS4zMi0uMDUuMjQtLjAxaC4xNmwuMDYuMDFoOC4xNnYtLjgzSDYuMThsLS4wMS0yLjc1LS4wMi0uMzcuMDUtLjM0LjExLS4zMS4xNy0uMjguMjUtLjI2LjMxLS4yMy4zOC0uMi40NC0uMTguNTEtLjE1LjU4LS4xMi42NC0uMS43MS0uMDYuNzctLjA0Ljg0LS4wMiAxLjI3LjA1em0tNi4zIDEuOThsLS4yMy4zMy0uMDguNDEuMDguNDEuMjMuMzQuMzMuMjIuNDEuMDkuNDEtLjA5LjMzLS4yMi4yMy0uMzQuMDgtLjQxLS4wOC0uNDEtLjIzLS4zMy0uMzMtLjIyLS40MS0uMDktLjQxLjA5em0xMy4wOSAzLjk1bC4yOC4wNi4zMi4xMi4zNS4xOC4zNi4yNy4zNi4zNS4zNS40Ny4zMi41OS4yOC43My4yMS44OC4xNCAxLjA0LjA1IDEuMjMtLjA2IDEuMjMtLjE2IDEuMDQtLjI0Ljg2LS4zMi43MS0uMzYuNTctLjQuNDUtLjQyLjMzLS40Mi4yNC0uNC4xNi0uMzYuMDktLjMyLjA1LS4yNC4wMi0uMTYtLjAxaC04LjIydi44Mmg1Ljg0bC4wMSAyLjc2LjAyLjM2LS4wNS4zNC0uMTEuMzEtLjE3LjI5LS4yNS4yNS0uMzEuMjQtLjM4LjItLjQ0LjE3LS41MS4xNS0uNTguMTMtLjY0LjA5LS43MS4wNy0uNzcuMDQtLjg0LjAxLTEuMjctLjA0LTEuMDctLjE0LS45LS4yLS43My0uMjUtLjU5LS4zLS40NS0uMzMtLjM0LS4zNC0uMjUtLjM0LS4xNi0uMzMtLjEtLjMtLjA0LS4yNS0uMDItLjIuMDEtLjEzdi01LjM0bC4wNS0uNjQuMTMtLjU0LjIxLS40Ni4yNi0uMzguMy0uMzIuMzMtLjI0LjM1LS4yLjM1LS4xNC4zMy0uMS4zLS4wNi4yNi0uMDQuMjEtLjAyLjEzLS4wMWg1Ljg0bC42OS0uMDUuNTktLjE0LjUtLjIxLjQxLS4yOC4zMy0uMzIuMjctLjM1LjItLjM2LjE1LS4zNi4xLS4zNS4wNy0uMzIuMDQtLjI4LjAyLS4yMVY2LjA3aDIuMDlsLjE0LjAxem0tNi40NyAxNC4yNWwtLjIzLjMzLS4wOC40MS4wOC40MS4yMy4zMy4zMy4yMy40MS4wOC40MS0uMDguMzMtLjIzLjIzLS4zMy4wOC0uNDEtLjA4LS40MS0uMjMtLjMzLS4zMy0uMjMtLjQxLS4wOC0uNDEuMDh6Ii8+PC9zdmc+"/><text transform="scale(.1)" x="587.5" y="175" textLength="535" fill="#fff" font-weight="bold">PYTHON</text></g></svg>

docs/src/basic.md

@@ -16,11 +16,60 @@
       pip install lancedb
       ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-      ```shell
-      npm install vectordb
-      ```
+        ```shell
+        npm install @lancedb/lancedb
+        ```
+        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
+
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
+                return config;
+            }
+            })
+            ```
+
+        !!! note "Yarn users"
+
+            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
+
+            ```shell
+            yarn add apache-arrow
+            ```
+
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb
+        ```
+        !!! note "Bundling `vectordb` apps with Webpack"
+
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ vectordb: 'vectordb' })
+                return config;
+            }
+            })
+            ```
+
+        !!! note "Yarn users"
+
+            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
+
+            ```shell
+            yarn add apache-arrow
+            ```
 
 === "Rust"
 
@@ -44,6 +93,43 @@
 
     !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)"
 
+### Preview releases
+
+Stable releases are created about every 2 weeks. For the latest features and bug
+fixes, you can install the preview release. These releases receive the same
+level of testing as stable releases, but are not guaranteed to be available for
+more than 6 months after they are released. Once your application is stable, we
+recommend switching to stable releases.
+
+=== "Python"
+
+      ```shell
+      pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
+      ```
+
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```shell
+        npm install @lancedb/lancedb@preview
+        ```
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb@preview
+        ```
+
+=== "Rust"
+
+    We don't push preview releases to crates.io, but you can referent the tag
+    in GitHub within your Cargo dependencies:
+
+    ```toml
+    [dependencies]
+    lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" }
+    ```
+
 ## Connect to a database
 
 === "Python"
@@ -63,13 +149,22 @@
         use the same syntax as the asynchronous API.  To help with this migration we
         have created a [migration guide](migration.md) detailing the differences.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:import"
+    === "@lancedb/lancedb"
 
-    --8<-- "docs/src/basic_legacy.ts:open_db"
-    ```
+        ```typescript
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
+
+        --8<-- "nodejs/examples/basic.ts:connect"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:open_db"
+        ```
 
 === "Rust"
 
@@ -112,15 +207,23 @@ table.
     --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:create_table"
-    ```
+    === "@lancedb/lancedb"
 
-    If the table already exists, LanceDB will raise an error by default.
-    If you want to overwrite the table, you can pass in `mode="overwrite"`
-    to the `createTable` function.
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_table"
+        ```
+
+        If the table already exists, LanceDB will raise an error by default.
+        If you want to overwrite the table, you can pass in `mode:"overwrite"`
+        to the `createTable` function.
 
 === "Rust"
 
@@ -140,6 +243,9 @@ table.
 
 !!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
 
+!!! info "Automatic embedding generation with Embedding API"
+    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
+
 ### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
@@ -154,11 +260,22 @@ similar to a `CREATE TABLE` statement in SQL.
       --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
       ```
 
-=== "Typescript"
+    !!! note "You can define schema in Pydantic"
+        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
-    ```
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
 
 === "Rust"
 
@@ -177,11 +294,19 @@ Once created, you can open a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:open_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const tbl = await db.openTable("myTable");
+        ```
 
-    ```typescript
-    const tbl = await db.openTable("myTable");
-    ```
 
 === "Rust"
 
@@ -198,11 +323,18 @@ If you forget the name of your table, you can always get a listing of all table
     --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
-=== "Javascript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```javascript
-    console.log(await db.tableNames());
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:table_names"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        console.log(await db.tableNames());
+        ```
 
 === "Rust"
 
@@ -221,11 +353,18 @@ After a table has been created, you can always add more data to it as follows:
     --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:add"
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:add_data"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:add"
+        ```
 
 === "Rust"
 
@@ -246,11 +385,18 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 
     This returns a pandas DataFrame with the results.
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:search"
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:vector_search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:search"
+        ```
 
 === "Rust"
 
@@ -279,11 +425,18 @@ LanceDB allows you to create an ANN index on a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```{.typescript .ignore}
-    --8<-- "docs/src/basic_legacy.ts:create_index"
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_index"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```{.typescript .ignore}
+        --8<-- "docs/src/basic_legacy.ts:create_index"
+        ```
 
 === "Rust"
 
@@ -311,11 +464,19 @@ This can delete any number of rows that match the filter.
     --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
-    --8<-- "docs/src/basic_legacy.ts:delete"
-    ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:delete"
+        ```
 
 === "Rust"
 
@@ -332,9 +493,15 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [lancedb.table.Table.delete][]
 
-=== "Javascript"
+=== "Typescript[^1]"
 
-      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+    === "@lancedb/lancedb"
+
+        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
+
+    === "vectordb (deprecated)"
+
+        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
 === "Rust"
 
@@ -346,23 +513,31 @@ Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
-      ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-      ```
+    ```python
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+    ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      By default, if the table does not exist an exception is raised. To suppress this,
-      you can pass in `ignore_missing=True`.
+    This permanently removes the table and is not recoverable, unlike deleting rows.
+    By default, if the table does not exist an exception is raised. To suppress this,
+    you can pass in `ignore_missing=True`.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-      ```typescript
-      --8<-- "docs/src/basic_legacy.ts:drop_table"
-      ```
+    === "@lancedb/lancedb"
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
-      If the table does not exist an exception is raised.
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:drop_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:drop_table"
+        ```
+
+        This permanently removes the table and is not recoverable, unlike deleting rows.
+        If the table does not exist an exception is raised.
 
 === "Rust"
 
@@ -370,22 +545,40 @@ Use the `drop_table()` method on the database to remove a table.
     --8<-- "rust/lancedb/examples/simple.rs:drop_table"
     ```
 
-!!! note "Bundling `vectordb` apps with Webpack"
 
-    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+## Using the Embedding API
+You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
 
-    ```javascript
-    /** @type {import('next').NextConfig} */
-    module.exports = ({
-    webpack(config) {
-        config.externals.push({ vectordb: 'vectordb' })
-        return config;
-    }
-    })
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
     ```
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
+Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
+
+
 ## What's next
 
 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
 
 If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -1,6 +1,14 @@
 // --8<-- [start:import]
 import * as lancedb from "vectordb";
-import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
+import {
+  Schema,
+  Field,
+  Float32,
+  FixedSizeList,
+  Int32,
+  Float16,
+} from "apache-arrow";
+import * as arrow from "apache-arrow";
 // --8<-- [end:import]
 import * as fs from "fs";
 import { Table as ArrowTable, Utf8 } from "apache-arrow";
@@ -20,9 +28,33 @@ const example = async () => {
       { vector: [3.1, 4.1], item: "foo", price: 10.0 },
       { vector: [5.9, 26.5], item: "bar", price: 20.0 },
     ],
-    { writeMode: lancedb.WriteMode.Overwrite }
+    { writeMode: lancedb.WriteMode.Overwrite },
   );
   // --8<-- [end:create_table]
+  {
+    // --8<-- [start:create_table_with_schema]
+    const schema = new arrow.Schema([
+      new arrow.Field(
+        "vector",
+        new arrow.FixedSizeList(
+          2,
+          new arrow.Field("item", new arrow.Float32(), true),
+        ),
+      ),
+      new arrow.Field("item", new arrow.Utf8(), true),
+      new arrow.Field("price", new arrow.Float32(), true),
+    ]);
+    const data = [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ];
+    const tbl = await db.createTable({
+      name: "myTableWithSchema",
+      data,
+      schema,
+    });
+    // --8<-- [end:create_table_with_schema]
+  }
 
   // --8<-- [start:add]
   const newData = Array.from({ length: 500 }, (_, i) => ({
@@ -42,33 +74,35 @@ const example = async () => {
   // --8<-- [end:create_index]
 
   // --8<-- [start:create_empty_table]
-  const schema = new Schema([
-    new Field("id", new Int32()),
-    new Field("name", new Utf8()),
+  const schema = new arrow.Schema([
+    new arrow.Field("id", new arrow.Int32()),
+    new arrow.Field("name", new arrow.Utf8()),
   ]);
+
   const empty_tbl = await db.createTable({ name: "empty_table", schema });
   // --8<-- [end:create_empty_table]
-
-  // --8<-- [start:create_f16_table]
-  const dim = 16
-  const total = 10
-  const f16_schema = new Schema([
-      new Field('id', new Int32()),
+  {
+    // --8<-- [start:create_f16_table]
+    const dim = 16;
+    const total = 10;
+    const schema = new Schema([
+      new Field("id", new Int32()),
       new Field(
-        'vector',
-        new FixedSizeList(dim, new Field('item', new Float16(), true)),
-        false
-      )
-    ])
-  const data = lancedb.makeArrowTable(
+        "vector",
+        new FixedSizeList(dim, new Field("item", new Float16(), true)),
+        false,
+      ),
+    ]);
+    const data = lancedb.makeArrowTable(
       Array.from(Array(total), (_, i) => ({
         id: i,
-        vector: Array.from(Array(dim), Math.random)
+        vector: Array.from(Array(dim), Math.random),
       })),
-      { f16_schema }
-    )
-  const table = await db.createTable('f16_tbl', data)
-  // --8<-- [end:create_f16_table]
+      { schema },
+    );
+    const table = await db.createTable("f16_tbl", data);
+    // --8<-- [end:create_f16_table]
+  }
 
   // --8<-- [start:search]
   const query = await tbl.search([100, 100]).limit(2).execute();

docs/src/cli_config.md

@@ -1,51 +0,0 @@
-
-# CLI & Config
-
-## LanceDB CLI
-Once lanceDB is installed, you can access the CLI using `lancedb` command on the console.
-
-```
-lancedb
-```
-
-This lists out all the various command-line options available. You can get the usage or help for a particular command.
-
-```
-lancedb {command} --help
-```
-
-## LanceDB config
-LanceDB uses a global config file to store certain settings. These settings are configurable using the lanceDB cli.
-To view your config settings, you can use:
-
-```
-lancedb config
-```
-
-These config parameters can be tuned using the cli.
-
-```
-lancedb {config_name} --{argument}
-```
-
-## LanceDB Opt-in Diagnostics
-When enabled, LanceDB will send anonymous events to help us improve LanceDB. These diagnostics are used only for error reporting and no data is collected. Error & stats allow us to automate certain aspects of bug reporting, prioritization of fixes and feature requests.
-These diagnostics are opt-in and can be enabled or disabled using the `lancedb diagnostics` command. These are enabled by default.
-
-### Get usage help
-
-```
-lancedb diagnostics --help
-```
-
-### Disable diagnostics
-
-```
-lancedb diagnostics --disabled
-```
-
-### Enable diagnostics
-
-```
-lancedb diagnostics --enabled
-```

docs/src/cloud/rest.md

@@ -0,0 +1 @@
+!!swagger ../../openapi.yml!!

docs/src/embeddings/custom_embedding_function.md

@@ -15,198 +15,226 @@ There is another optional layer of abstraction available: `TextEmbeddingFunction
 
 Let's implement `SentenceTransformerEmbeddings` class. All you need to do is implement the `generate_embeddings()` and `ndims` function to handle the input types you expect and register the class in the global `EmbeddingFunctionRegistry`
 
-```python
-from lancedb.embeddings import register
-from lancedb.util import attempt_import_or_raise
 
-@register("sentence-transformers")
-class SentenceTransformerEmbeddings(TextEmbeddingFunction):
-    name: str = "all-MiniLM-L6-v2"
-    # set more default instance vars like device, etc.
+=== "Python"
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._ndims = None
-    
-    def generate_embeddings(self, texts):
-        return self._embedding_model().encode(list(texts), ...).tolist()
+    ```python
+    from lancedb.embeddings import register
+    from lancedb.util import attempt_import_or_raise
 
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = len(self.generate_embeddings("foo")[0])
-        return self._ndims
+    @register("sentence-transformers")
+    class SentenceTransformerEmbeddings(TextEmbeddingFunction):
+        name: str = "all-MiniLM-L6-v2"
+        # set more default instance vars like device, etc.
 
-    @cached(cache={}) 
-    def _embedding_model(self):
-        return sentence_transformers.SentenceTransformer(name)
-```
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            self._ndims = None
 
-This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and defaul settings.
+        def generate_embeddings(self, texts):
+            return self._embedding_model().encode(list(texts), ...).tolist()
+
+        def ndims(self):
+            if self._ndims is None:
+                self._ndims = len(self.generate_embeddings("foo")[0])
+            return self._ndims
+
+        @cached(cache={})
+        def _embedding_model(self):
+            return sentence_transformers.SentenceTransformer(name)
+    ```
+
+=== "TypeScript"
+
+    ```ts
+    --8<--- "nodejs/examples/custom_embedding_function.ts:imports"
+
+    --8<--- "nodejs/examples/custom_embedding_function.ts:embedding_impl"
+    ```
+
+
+This is a stripped down version of our implementation of `SentenceTransformerEmbeddings` that removes certain optimizations and default settings.
 
 Now you can use this embedding function to create your table schema and that's it! you can then ingest data and run queries without manually vectorizing the inputs.
 
-```python
-from lancedb.pydantic import LanceModel, Vector
+=== "Python"
 
-registry = EmbeddingFunctionRegistry.get_instance()
-stransformer = registry.get("sentence-transformers").create()
+    ```python
+    from lancedb.pydantic import LanceModel, Vector
 
-class TextModelSchema(LanceModel):
-    vector: Vector(stransformer.ndims) = stransformer.VectorField()
-    text: str = stransformer.SourceField()
+    registry = EmbeddingFunctionRegistry.get_instance()
+    stransformer = registry.get("sentence-transformers").create()
 
-tbl = db.create_table("table", schema=TextModelSchema)
+    class TextModelSchema(LanceModel):
+        vector: Vector(stransformer.ndims) = stransformer.VectorField()
+        text: str = stransformer.SourceField()
 
-tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
-result = tbl.search("world").limit(5)
-```
+    tbl = db.create_table("table", schema=TextModelSchema)
 
-NOTE:
+    tbl.add(pd.DataFrame({"text": ["halo", "world"]}))
+    result = tbl.search("world").limit(5)
+    ```
 
-You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
+=== "TypeScript"
+
+    ```ts
+    --8<--- "nodejs/examples/custom_embedding_function.ts:call_custom_function"
+    ```
+
+!!! note
+
+    You can always implement the `EmbeddingFunction` interface directly if you want or need to, `TextEmbeddingFunction` just makes it much simpler and faster for you to do so, by setting up the boiler plat for text-specific use case
 
 ## Multi-modal embedding function example
-You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support. LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
+You can also use the `EmbeddingFunction` interface to implement more complex workflows such as multi-modal embedding function support.
 
-```python
-@register("open-clip")
-class OpenClipEmbeddings(EmbeddingFunction):
-    name: str = "ViT-B-32"
-    pretrained: str = "laion2b_s34b_b79k"
-    device: str = "cpu"
-    batch_size: int = 64
-    normalize: bool = True
-    _model = PrivateAttr()
-    _preprocess = PrivateAttr()
-    _tokenizer = PrivateAttr()
+=== "Python"
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
-        model, _, preprocess = open_clip.create_model_and_transforms(
-            self.name, pretrained=self.pretrained
-        )
-        model.to(self.device)
-        self._model, self._preprocess = model, preprocess
-        self._tokenizer = open_clip.get_tokenizer(self.name)
-        self._ndims = None
+    LanceDB implements `OpenClipEmeddingFunction` class that suppports multi-modal seach. Here's the implementation that you can use as a reference to build your own multi-modal embedding functions.
 
-    def ndims(self):
-        if self._ndims is None:
-            self._ndims = self.generate_text_embeddings("foo").shape[0]
-        return self._ndims
+    ```python
+    @register("open-clip")
+    class OpenClipEmbeddings(EmbeddingFunction):
+        name: str = "ViT-B-32"
+        pretrained: str = "laion2b_s34b_b79k"
+        device: str = "cpu"
+        batch_size: int = 64
+        normalize: bool = True
+        _model = PrivateAttr()
+        _preprocess = PrivateAttr()
+        _tokenizer = PrivateAttr()
 
-    def compute_query_embeddings(
-        self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
-    ) -> List[np.ndarray]:
-        """
-        Compute the embeddings for a given user query
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+            model, _, preprocess = open_clip.create_model_and_transforms(
+                self.name, pretrained=self.pretrained
+            )
+            model.to(self.device)
+            self._model, self._preprocess = model, preprocess
+            self._tokenizer = open_clip.get_tokenizer(self.name)
+            self._ndims = None
 
-        Parameters
-        ----------
-        query : Union[str, PIL.Image.Image]
-            The query to embed. A query can be either text or an image.
-        """
-        if isinstance(query, str):
-            return [self.generate_text_embeddings(query)]
-        else:
+        def ndims(self):
+            if self._ndims is None:
+                self._ndims = self.generate_text_embeddings("foo").shape[0]
+            return self._ndims
+
+        def compute_query_embeddings(
+            self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
+        ) -> List[np.ndarray]:
+            """
+            Compute the embeddings for a given user query
+
+            Parameters
+            ----------
+            query : Union[str, PIL.Image.Image]
+                The query to embed. A query can be either text or an image.
+            """
+            if isinstance(query, str):
+                return [self.generate_text_embeddings(query)]
+            else:
+                PIL = attempt_import_or_raise("PIL", "pillow")
+                if isinstance(query, PIL.Image.Image):
+                    return [self.generate_image_embedding(query)]
+                else:
+                    raise TypeError("OpenClip supports str or PIL Image as query")
+
+        def generate_text_embeddings(self, text: str) -> np.ndarray:
+            torch = attempt_import_or_raise("torch")
+            text = self.sanitize_input(text)
+            text = self._tokenizer(text)
+            text.to(self.device)
+            with torch.no_grad():
+                text_features = self._model.encode_text(text.to(self.device))
+                if self.normalize:
+                    text_features /= text_features.norm(dim=-1, keepdim=True)
+                return text_features.cpu().numpy().squeeze()
+
+        def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
+            """
+            Sanitize the input to the embedding function.
+            """
+            if isinstance(images, (str, bytes)):
+                images = [images]
+            elif isinstance(images, pa.Array):
+                images = images.to_pylist()
+            elif isinstance(images, pa.ChunkedArray):
+                images = images.combine_chunks().to_pylist()
+            return images
+
+        def compute_source_embeddings(
+            self, images: IMAGES, *args, **kwargs
+        ) -> List[np.array]:
+            """
+            Get the embeddings for the given images
+            """
+            images = self.sanitize_input(images)
+            embeddings = []
+            for i in range(0, len(images), self.batch_size):
+                j = min(i + self.batch_size, len(images))
+                batch = images[i:j]
+                embeddings.extend(self._parallel_get(batch))
+            return embeddings
+
+        def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
+            """
+            Issue concurrent requests to retrieve the image data
+            """
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                futures = [
+                    executor.submit(self.generate_image_embedding, image)
+                    for image in images
+                ]
+                return [future.result() for future in futures]
+
+        def generate_image_embedding(
+            self, image: Union[str, bytes, "PIL.Image.Image"]
+        ) -> np.ndarray:
+            """
+            Generate the embedding for a single image
+
+            Parameters
+            ----------
+            image : Union[str, bytes, PIL.Image.Image]
+                The image to embed. If the image is a str, it is treated as a uri.
+                If the image is bytes, it is treated as the raw image bytes.
+            """
+            torch = attempt_import_or_raise("torch")
+            # TODO handle retry and errors for https
+            image = self._to_pil(image)
+            image = self._preprocess(image).unsqueeze(0)
+            with torch.no_grad():
+                return self._encode_and_normalize_image(image)
+
+        def _to_pil(self, image: Union[str, bytes]):
             PIL = attempt_import_or_raise("PIL", "pillow")
-            if isinstance(query, PIL.Image.Image):
-                return [self.generate_image_embedding(query)]
-            else:
-                raise TypeError("OpenClip supports str or PIL Image as query")
+            if isinstance(image, bytes):
+                return PIL.Image.open(io.BytesIO(image))
+            if isinstance(image, PIL.Image.Image):
+                return image
+            elif isinstance(image, str):
+                parsed = urlparse.urlparse(image)
+                # TODO handle drive letter on windows.
+                if parsed.scheme == "file":
+                    return PIL.Image.open(parsed.path)
+                elif parsed.scheme == "":
+                    return PIL.Image.open(image if os.name == "nt" else parsed.path)
+                elif parsed.scheme.startswith("http"):
+                    return PIL.Image.open(io.BytesIO(url_retrieve(image)))
+                else:
+                    raise NotImplementedError("Only local and http(s) urls are supported")
 
-    def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = attempt_import_or_raise("torch")
-        text = self.sanitize_input(text)
-        text = self._tokenizer(text)
-        text.to(self.device)
-        with torch.no_grad():
-            text_features = self._model.encode_text(text.to(self.device))
+        def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
+            """
+            encode a single image tensor and optionally normalize the output
+            """
+            image_features = self._model.encode_image(image_tensor)
             if self.normalize:
-                text_features /= text_features.norm(dim=-1, keepdim=True)
-            return text_features.cpu().numpy().squeeze()
+                image_features /= image_features.norm(dim=-1, keepdim=True)
+            return image_features.cpu().numpy().squeeze()
+    ```
 
-    def sanitize_input(self, images: IMAGES) -> Union[List[bytes], np.ndarray]:
-        """
-        Sanitize the input to the embedding function.
-        """
-        if isinstance(images, (str, bytes)):
-            images = [images]
-        elif isinstance(images, pa.Array):
-            images = images.to_pylist()
-        elif isinstance(images, pa.ChunkedArray):
-            images = images.combine_chunks().to_pylist()
-        return images
+=== "TypeScript"
 
-    def compute_source_embeddings(
-        self, images: IMAGES, *args, **kwargs
-    ) -> List[np.array]:
-        """
-        Get the embeddings for the given images
-        """
-        images = self.sanitize_input(images)
-        embeddings = []
-        for i in range(0, len(images), self.batch_size):
-            j = min(i + self.batch_size, len(images))
-            batch = images[i:j]
-            embeddings.extend(self._parallel_get(batch))
-        return embeddings
-
-    def _parallel_get(self, images: Union[List[str], List[bytes]]) -> List[np.ndarray]:
-        """
-        Issue concurrent requests to retrieve the image data
-        """
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [
-                executor.submit(self.generate_image_embedding, image)
-                for image in images
-            ]
-            return [future.result() for future in futures]
-
-    def generate_image_embedding(
-        self, image: Union[str, bytes, "PIL.Image.Image"]
-    ) -> np.ndarray:
-        """
-        Generate the embedding for a single image
-
-        Parameters
-        ----------
-        image : Union[str, bytes, PIL.Image.Image]
-            The image to embed. If the image is a str, it is treated as a uri.
-            If the image is bytes, it is treated as the raw image bytes.
-        """
-        torch = attempt_import_or_raise("torch")
-        # TODO handle retry and errors for https
-        image = self._to_pil(image)
-        image = self._preprocess(image).unsqueeze(0)
-        with torch.no_grad():
-            return self._encode_and_normalize_image(image)
-
-    def _to_pil(self, image: Union[str, bytes]):
-        PIL = attempt_import_or_raise("PIL", "pillow")
-        if isinstance(image, bytes):
-            return PIL.Image.open(io.BytesIO(image))
-        if isinstance(image, PIL.Image.Image):
-            return image
-        elif isinstance(image, str):
-            parsed = urlparse.urlparse(image)
-            # TODO handle drive letter on windows.
-            if parsed.scheme == "file":
-                return PIL.Image.open(parsed.path)
-            elif parsed.scheme == "":
-                return PIL.Image.open(image if os.name == "nt" else parsed.path)
-            elif parsed.scheme.startswith("http"):
-                return PIL.Image.open(io.BytesIO(url_retrieve(image)))
-            else:
-                raise NotImplementedError("Only local and http(s) urls are supported")
-
-    def _encode_and_normalize_image(self, image_tensor: "torch.Tensor"):
-        """
-        encode a single image tensor and optionally normalize the output
-        """
-        image_features = self._model.encode_image(image_tensor)
-        if self.normalize:
-            image_features /= image_features.norm(dim=-1, keepdim=True)
-        return image_features.cpu().numpy().squeeze()
-```
+    Coming Soon! See this [issue](https://github.com/lancedb/lancedb/issues/1482) to track the status!

docs/src/embeddings/default_embedding_functions.md

@@ -17,6 +17,7 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
 | `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
 | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
+| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository |
 
 
 ??? "Check out available sentence-transformer models here!"
@@ -154,9 +155,12 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 !!! note "BAAI Embeddings example"
     Here is an example that uses BAAI embedding model from the HuggingFace Hub [supported models](https://huggingface.co/models?library=sentence-transformers)
     ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
     db = lancedb.connect("/tmp/db")
-    registry = EmbeddingFunctionRegistry.get_instance()
-    model = registry.get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
 
     class Words(LanceModel):
         text: str = model.SourceField()
@@ -165,7 +169,7 @@ Allows you to set parameters when registering a `sentence-transformers` object.
     table = db.create_table("words", schema=Words)
     table.add(
         [
-            {"text": "hello world"}
+            {"text": "hello world"},
             {"text": "goodbye world"}
         ]
     )
@@ -177,6 +181,70 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models.
 
 
+### Huggingface embedding models
+We offer support for all huggingface models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")` 
+
+Example usage - 
+```python
+import lancedb
+import pandas as pd
+
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+model = get_registry().get("huggingface").create(name='facebook/bart-base')
+
+class Words(LanceModel):
+    text: str = model.SourceField()
+    vector: Vector(model.ndims()) = model.VectorField()
+
+df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
+table = db.create_table("greets", schema=Words)
+table.add(df)
+query = "old greeting"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+
+### Ollama embeddings
+Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) python library. More details:
+
+- [Ollama docs on embeddings](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings)
+- [Ollama blog on embeddings](https://ollama.com/blog/embedding-models)
+
+| Parameter              | Type                       | Default Value            | Description                                                                                                                                    |
+|------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
+| `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
+| `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
+| `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("ollama").create(name="nomic-embed-text")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add([
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+])
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```
+
+
 ### OpenAI embeddings
 LanceDB registers the OpenAI embeddings function in the registry by default, as `openai`. Below are the parameters that you can customize when creating the instances:
 
@@ -187,18 +255,21 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
 
 
 ```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
 db = lancedb.connect("/tmp/db")
-registry = EmbeddingFunctionRegistry.get_instance()
-func = registry.get("openai").create()
+func = get_registry().get("openai").create(name="text-embedding-ada-002")
 
 class Words(LanceModel):
     text: str = func.SourceField()
     vector: Vector(func.ndims()) = func.VectorField()
 
-table = db.create_table("words", schema=Words)
+table = db.create_table("words", schema=Words, mode="overwrite")
 table.add(
     [
-        {"text": "hello world"}
+        {"text": "hello world"},
         {"text": "goodbye world"}
     ]
     )
@@ -295,6 +366,108 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```
 
+### Cohere Embeddings
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+```
+    * embed-english-v3.0
+    * embed-multilingual-v3.0
+    * embed-english-light-v3.0
+    * embed-multilingual-light-v3.0
+    * embed-english-v2.0
+    * embed-english-light-v2.0
+    * embed-multilingual-v2.0
+```
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+    
+    ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import EmbeddingFunctionRegistry
+
+        cohere = EmbeddingFunctionRegistry
+            .get_instance()
+            .get("cohere")
+            .create(name="embed-multilingual-v2.0")
+
+        class TextModel(LanceModel):
+            text: str = cohere.SourceField()
+            vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+        data = [ { "text": "hello world" },
+                { "text": "goodbye world" }]
+
+        db = lancedb.connect("~/.lancedb")
+        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+        tbl.add(data)
+    ```
+
+### Jina Embeddings
+Jina embeddings are used to generate embeddings for text and image data.
+You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
+
+You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
+
+
+    class TextModel(LanceModel):
+        text: str = jina_embed.SourceField()
+        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
+
+
+    data = [{"text": "hello world"},
+            {"text": "goodbye world"}]
+
+    db = lancedb.connect("~/.lancedb-2")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+```
+
 ### AWS Bedrock Text Embedding Functions
 AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
 You can do so by using `awscli` and also add your session_token:
@@ -327,6 +500,10 @@ Supported parameters (to be passed in `create` method) are:
 Usage Example:
 
 ```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
 model = get_registry().get("bedrock-text").create()
 
 class TextModel(LanceModel):
@@ -341,6 +518,82 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```
 
+# IBM watsonx.ai Embeddings
+
+Generate text embeddings using IBM's watsonx.ai platform.
+
+## Supported Models
+
+You can find a list of supported models at [IBM watsonx.ai Documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx). The currently supported model names are:
+
+- `ibm/slate-125m-english-rtrvr`
+- `ibm/slate-30m-english-rtrvr`
+- `sentence-transformers/all-minilm-l12-v2`
+- `intfloat/multilingual-e5-large`
+
+## Parameters
+
+The following parameters can be passed to the `create` method:
+
+| Parameter  | Type     | Default Value                    | Description                                               |
+|------------|----------|----------------------------------|-----------------------------------------------------------|
+| name       | str      | "ibm/slate-125m-english-rtrvr"   | The model ID of the watsonx.ai model to use               |
+| api_key    | str      | None                             | Optional IBM Cloud API key (or set `WATSONX_API_KEY`)     |
+| project_id | str      | None                             | Optional watsonx project ID (or set `WATSONX_PROJECT_ID`) |
+| url        | str      | None                             | Optional custom URL for the watsonx.ai instance           |
+| params     | dict     | None                             | Optional additional parameters for the embedding model    |
+
+## Usage Example
+
+First, the watsonx.ai library is an optional dependency, so must be installed seperately:
+
+```
+pip install ibm-watsonx-ai
+```
+
+Optionally set environment variables (if not passing credentials to `create` directly):
+
+```sh
+export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
+export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
+```
+
+```python
+import os
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import EmbeddingFunctionRegistry
+
+watsonx_embed = EmbeddingFunctionRegistry
+  .get_instance()
+  .get("watsonx")
+  .create(
+    name="ibm/slate-125m-english-rtrvr",
+    # Uncomment and set these if not using environment variables
+    # api_key="your_api_key_here",
+    # project_id="your_project_id_here",
+    # url="your_watsonx_url_here",
+    # params={...},
+  )
+
+class TextModel(LanceModel):
+    text: str = watsonx_embed.SourceField()
+    vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"},
+]
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
+
+tbl.add(data)
+
+rs = tbl.search("hello").limit(1).to_pandas()
+print(rs)
+```
+
 ## Multi-modal embedding functions
 Multi-modal embedding functions allow you to query your table using both images and text.
 
@@ -361,10 +614,12 @@ This embedding function supports ingesting images as both bytes and urls. You ca
     LanceDB supports ingesting images directly from accessible links.
 
 ```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
 
 db = lancedb.connect(tmp_path)
-registry = EmbeddingFunctionRegistry.get_instance()
-func = registry.get("open-clip").create()
+func = get_registry.get("open-clip").create()
 
 class Images(LanceModel):
     label: str
@@ -386,7 +641,7 @@ uris = [
 # get each uri as bytes
 image_bytes = [requests.get(uri).content for uri in uris]
 table.add(
-    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
 )
 ```
 Now we can search using text from both the default vector column and the custom vector column
@@ -439,9 +694,12 @@ This function is registered as `imagebind` and supports Audio, Video and Text mo
 Below is an example demonstrating how the API works:
 
 ```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+
 db = lancedb.connect(tmp_path)
-registry = EmbeddingFunctionRegistry.get_instance()
-func = registry.get("imagebind").create()
+func = get_registry.get("imagebind").create()
 
 class ImageBindModel(LanceModel):
     text: str
@@ -489,3 +747,54 @@ print(actual.text == "bird")
 ```
 
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).
+
+### Jina Embeddings
+Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
+under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import requests
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+    import pandas as pd
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    db = lancedb.connect("~/.lancedb")
+    func = get_registry().get("jina").create()
+
+
+    class Images(LanceModel):
+        label: str
+        image_uri: str = func.SourceField()  # image uri as the source
+        image_bytes: bytes = func.SourceField()  # image bytes as the source
+        vector: Vector(func.ndims()) = func.VectorField()  # vector column
+        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
+
+
+    table = db.create_table("images", schema=Images)
+    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+    ]
+    # get each uri as bytes
+    image_bytes = [requests.get(uri).content for uri in uris]
+    table.add(
+      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
+    )
+```

docs/src/embeddings/embedding_functions.md

@@ -2,9 +2,12 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
+!!! Note "LanceDB cloud doesn't support embedding functions yet"
+    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
+
 !!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself.
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function
     and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
     table metadata and have LanceDB automatically take care of regenerating the embeddings.
 
@@ -13,7 +16,7 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
 === "Python"
     In the LanceDB python SDK, we define a global embedding function registry with
-    many different embedding models and even more coming soon. 
+    many different embedding models and even more coming soon.
     Here's let's an implementation of CLIP as example.
 
     ```python
@@ -23,20 +26,35 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
     clip = registry.get("open-clip").create()
     ```
 
-    You can also define your own embedding function by implementing the `EmbeddingFunction` 
+    You can also define your own embedding function by implementing the `EmbeddingFunction`
     abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
 
-=== "JavaScript""
+=== "TypeScript"
     In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
     embedding function is available.
 
     ```javascript
-    const lancedb = require("vectordb");
+    import * as lancedb from '@lancedb/lancedb'
+    import { getRegistry } from '@lancedb/lancedb/embeddings'
 
     // You need to provide an OpenAI API key
     const apiKey = "sk-..."
     // The embedding function will create embeddings for the 'text' column
-    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    const func = getRegistry().get("openai").create({apiKey})
+    ```
+=== "Rust"
+    In the Rust SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
+
+    ```toml
+    // Make sure to include the `openai` feature
+    [dependencies]
+    lancedb = {version = "*", features = ["openai"]}
+    ```
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
     ```
 
 ## 2. Define the data model or schema
@@ -46,20 +64,20 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
     ```python
     class Pets(LanceModel):
-        vector: Vector(clip.ndims) = clip.VectorField()
+        vector: Vector(clip.ndims()) = clip.VectorField()
         image_uri: str = clip.SourceField()
     ```
 
     `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-=== "JavaScript"
+=== "TypeScript"
 
     For the TypeScript SDK, a schema can be inferred from input data, or an explicit
     Arrow schema can be provided.
 
 ## 3. Create table and add data
 
-Now that we have chosen/defined our embedding function and the schema, 
+Now that we have chosen/defined our embedding function and the schema,
 we can create the table and ingest data without needing to explicitly generate
 the embeddings at all:
 
@@ -71,17 +89,26 @@ the embeddings at all:
     table.add([{"image_uri": u} for u in uris])
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
+    === "@lancedb/lancedb"
 
-    const table = await db.createTable("vectors", data, embedding)
-    ```
+        ```ts
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const db = await lancedb.connect("data/sample-lancedb");
+        const data = [
+            { text: "pepperoni"},
+            { text: "pineapple"}
+        ]
+
+        const table = await db.createTable("vectors", data, embedding)
+        ```
 
 ## 4. Querying your table
 Not only can you forget about the embeddings during ingestion, you also don't
@@ -94,8 +121,8 @@ need to worry about it when you query the table:
     ```python
     results = (
         table.search("dog")
-        .limit(10)
-        .to_pandas()
+            .limit(10)
+            .to_pandas()
     )
     ```
 
@@ -106,22 +133,32 @@ need to worry about it when you query the table:
     query_image = Image.open(p)
     results = (
         table.search(query_image)
-        .limit(10)
-        .to_pandas()
+            .limit(10)
+            .to_pandas()
     )
     ```
 
     Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
-=== "JavaScript"
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        const results = await table.search("What's the best pizza topping?")
+            .limit(10)
+            .toArray()
+        ```
+
+    === "vectordb (deprecated)
+
+        ```ts
+        const results = await table
+            .search("What's the best pizza topping?")
+            .limit(10)
+            .execute()
+        ```
 
-    ```javascript
-    const results = await table
-    .search("What's the best pizza topping?")
-    .limit(10)
-    .execute()
-    ```    
-    
     The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 
 ---
@@ -149,7 +186,7 @@ You can also use the integration for adding utility operations in the schema. Fo
 
 ```python
 class Pets(LanceModel):
-    vector: Vector(clip.ndims) = clip.VectorField()
+    vector: Vector(clip.ndims()) = clip.VectorField()
     image_uri: str = clip.SourceField()
 
     @property
@@ -166,4 +203,4 @@ rs[2].image
 ![](../assets/dog_clip_output.png)
 
 Now that you have the basic idea about LanceDB embedding functions and the embedding function registry,
-let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).
+let's dive deeper into defining your own [custom functions](./custom_embedding_function.md).

docs/src/embeddings/index.md

@@ -1,14 +1,134 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
-This makes them a very powerful tool for machine learning practitioners. 
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio.
+This makes them a very powerful tool for machine learning practitioners.
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs
 (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
 LanceDB supports 3 methods of working with embeddings.
 
 1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
 2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+3. You can define your own [custom embedding function](./custom_embedding_function.md)
    that extends the default embedding functions.
 
 For python users, there is also a legacy [with_embeddings API](./legacy.md).
-It is retained for compatibility and will be removed in a future version.
+It is retained for compatibility and will be removed in a future version.
+
+## Quickstart
+
+To get started with embeddings, you can use the built-in embedding functions.
+
+### OpenAI Embedding function
+
+LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+        )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    --8<--- "nodejs/examples/embedding.ts:imports"
+    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<--- "rust/lancedb/examples/openai.rs:imports"
+    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
+### Sentence Transformers Embedding function
+LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
+
+=== "Python"
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+    class Words(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("words", schema=Words)
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+    )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    Coming Soon!
+
+=== "Rust"
+
+    Coming Soon!
+
+### Jina Embeddings
+
+LanceDB registers the JinaAI embeddings function in the registry as `jina`. You can pass any supported model name to the `create`. By default it uses `"jina-clip-v1"`.
+`jina-clip-v1` can handle both text and images and other models only support `text`.
+
+You need to pass `JINA_API_KEY` in the environment variable or pass it as `api_key` to `create` method.
+
+```python
+import os
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+os.environ['JINA_API_KEY'] = "jina_*"
+
+db = lancedb.connect("/tmp/db")
+func = get_registry().get("jina").create(name="jina-clip-v1")
+
+class Words(LanceModel):
+    text: str = func.SourceField()
+    vector: Vector(func.ndims()) = func.VectorField()
+
+table = db.create_table("words", schema=Words, mode="overwrite")
+table.add(
+    [
+        {"text": "hello world"},
+        {"text": "goodbye world"}
+    ]
+    )
+
+query = "greetings"
+actual = table.search(query).limit(1).to_pydantic(Words)[0]
+print(actual.text)
+```

docs/src/examples/index.md

@@ -10,7 +10,7 @@ LanceDB provides language APIs, allowing you to embed a database in your languag
 
 ## Applications powered by LanceDB
 
-| Project Name                                        | Description                                                                                                          | Screenshot                                |
-|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
-| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds                  | ![YOLOExplorer](https://github.com/lancedb/vectordb-recipes/assets/15766192/ae513a29-8f15-4e0b-99a1-ccd8272b6131) |
-| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. | ![Chatbot](../assets/vercel-template.gif)    |
+| Project Name | Description |
+| --- | --- |
+| **Ultralytics Explorer 🚀**<br>[![Ultralytics](https://img.shields.io/badge/Ultralytics-Docs-green?labelColor=0f3bc4&style=flat-square&logo=https://cdn.prod.website-files.com/646dd1f1a3703e451ba81ecc/64994922cf2a6385a4bf4489_UltralyticsYOLO_mark_blue.svg&link=https://docs.ultralytics.com/datasets/explorer/)](https://docs.ultralytics.com/datasets/explorer/)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ultralytics/ultralytics/blob/main/docs/en/datasets/explorer/explorer.ipynb) |  - 🔍 **Explore CV Datasets**: Semantic search, SQL queries, vector similarity, natural language.<br>- 🖥️ **GUI & Python API**: Seamless dataset interaction.<br>- ⚡ **Efficient & Scalable**: Leverages LanceDB for large datasets.<br>- 📊 **Detailed Analysis**: Easily analyze data patterns.<br>- 🌐 **Browser GUI Demo**: Create embeddings, search images, run queries. |
+| **Website Chatbot🤖**<br>[![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/lancedb-vercel-chatbot)<br>[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&amp;env=OPENAI_API_KEY&amp;envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&amp;project-name=lancedb-vercel-chatbot&amp;repository-name=lancedb-vercel-chatbot&amp;demo-title=LanceDB%20Chatbot%20Demo&amp;demo-description=Demo%20website%20chatbot%20with%20LanceDB.&amp;demo-url=https%3A%2F%2Flancedb.vercel.app&amp;demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png) | - 🌐 **Chatbot from Sitemap/Docs**: Create a chatbot using site or document context.<br>- 🚀 **Embed LanceDB in Next.js**: Lightweight, on-prem storage.<br>- 🧠 **AI-Powered Context Retrieval**: Efficiently access relevant data.<br>- 🔧 **Serverless & Native JS**: Seamless integration with Next.js.<br>- ⚡ **One-Click Deploy on Vercel**: Quick and easy setup.. |

docs/src/examples/python_examples/build_from_scratch.md

@@ -0,0 +1,13 @@
+# Build from Scratch with LanceDB 🚀
+
+Start building your GenAI applications from the ground up using LanceDB's efficient vector-based document retrieval capabilities! 📄
+
+#### Get Started in Minutes ⏱️
+
+These examples provide a solid foundation for building your own GenAI applications using LanceDB. Jump from idea to proof of concept quickly with applied examples. Get started and see what you can create! 💻
+
+| **Build From Scratch**                     | **Description**                                                                                              | **Links**                                                                                                                                                                                                                                                                                                                                                                                                                |
+|:-------------------------------------------|:-------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Build RAG from Scratch🚀💻**             | 📝 Create a **Retrieval-Augmented Generation** (RAG) model from scratch using LanceDB.                       | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/RAG-from-Scratch)<br>[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)]()                                                                                                                                  |
+| **Local RAG from Scratch with Llama3🔥💡** | 🐫 Build a local RAG model using **Llama3** and **LanceDB** for fast and efficient text generation.          | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Local-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Local-RAG-from-Scratch/rag.py)    |
+| **Multi-Head RAG from Scratch📚💻**        | 🤯 Develop a **Multi-Head RAG model** from scratch, enabling generation of text based on multiple documents. | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch)<br>[![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](https://github.com/lancedb/vectordb-recipes/tree/main/tutorials/Multi-Head-RAG-from-Scratch) |

docs/src/examples/python_examples/multimodal.md

@@ -0,0 +1,28 @@
+# Multimodal Search with LanceDB 🔍💡
+
+Experience the future of search with LanceDB's multimodal capabilities. Combine text and image queries to find the most relevant results in your corpus and unlock new possibilities! 🔓💡
+
+#### Explore the Future of Search 🚀
+
+Unlock the power of multimodal search with LanceDB, enabling efficient vector-based retrieval of text and image data! 📊💻
+
+
+
+| **Multimodal**  | **Description**  | **Links**  |
+|:----------------|:-----------------|:-----------|
+| **Multimodal CLIP: DiffusionDB 🌐💥**    | Revolutionize search with Multimodal CLIP and DiffusionDB, combining text and image understanding for a new dimension of discovery! 🔓 | [![GitHub](../../assets/github.svg)][Clip_diffusionDB_github] <br>[![Open In Collab](../../assets/colab.svg)][Clip_diffusionDB_colab] <br>[![Python](../../assets/python.svg)][Clip_diffusionDB_python] <br>[![Ghost](../../assets/ghost.svg)][Clip_diffusionDB_ghost] |
+| **Multimodal CLIP: Youtube Videos 📹👀** | Search Youtube videos using Multimodal CLIP, finding relevant content with ease and accuracy! 🎯                                                                                                                                                                | [![Github](../../assets/github.svg)][Clip_youtube_github]                                                                                                    <br>[![Open In Collab](../../assets/colab.svg)][Clip_youtube_colab]                                                                                           <br> [![Python](../../assets/python.svg)][Clip_youtube_python]         <br>[![Ghost](../../assets/ghost.svg)][Clip_youtube_python] |
+| **Multimodal Image + Text Search 📸🔍**               | Discover relevant documents and images with a single query, using LanceDB's multimodal search capabilities to bridge the gap between text and visuals! 🌉 | [![GitHub](../../assets/github.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search) <br>[![Open In Collab](../../assets/colab.svg)](https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.ipynb) <br> [![Python](../../assets/python.svg)](https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_search/main.py)<br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/) |
+| **Cambrian-1: Vision-Centric Image Exploration 🔍👀** | Dive into vision-centric exploration of images with Cambrian-1, powered by LanceDB's multimodal search to uncover new insights! 🔎                        | [![GitHub](../../assets/github.svg)](https://www.kaggle.com/code/prasantdixit/cambrian-1-vision-centric-exploration-of-images/)<br>[![Open In Collab](../../assets/colab.svg)]() <br> [![Python](../../assets/python.svg)]() <br> [![Ghost](../../assets/ghost.svg)](https://blog.lancedb.com/cambrian-1-vision-centric-exploration/)                                                                                                                                                                                                                   |
+
+
+[Clip_diffusionDB_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb
+[Clip_diffusionDB_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.ipynb
+[Clip_diffusionDB_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip_diffusiondb/main.py
+[Clip_diffusionDB_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/
+
+
+[Clip_youtube_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search
+[Clip_youtube_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.ipynb
+[Clip_youtube_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/multimodal_video_search/main.py
+[Clip_youtube_ghost]: https://blog.lancedb.com/multi-modal-ai-made-easy-with-lancedb-clip-5aaf8801c939/

docs/src/examples/python_examples/rag.md

@@ -0,0 +1,85 @@
+
+**🔍💡 RAG: Revolutionize Information Retrieval with LanceDB 🔓**
+====================================================================
+
+Unlock the full potential of Retrieval-Augmented Generation (RAG) with LanceDB, the ultimate solution for efficient vector-based information retrieval 📊. Input text queries and retrieve relevant documents with lightning-fast speed ⚡️ and accuracy ✅. Generate comprehensive answers by combining retrieved information, uncovering new insights 🔍 and connections.
+
+### Experience the Future of Search 🔄
+
+Experience the future of search with RAG, transforming information retrieval and answer generation. Apply RAG to various industries, streamlining processes 📈, saving time ⏰, and resources 💰. Stay ahead of the curve with innovative technology 🔝, powered by LanceDB. Discover the power of RAG with LanceDB and transform your industry with innovative solutions 💡.
+
+
+| **RAG**                                      | **Description**                                                                                                                                                  | **Links**                  |
+|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|                                                                                                                                                     
+| **RAG with Matryoshka Embeddings and LlamaIndex** 🪆🔗  | Utilize **Matryoshka embeddings** and **LlamaIndex** to improve the efficiency and accuracy of your RAG models. 📈✨ | [![Github](../../assets/github.svg)][matryoshka_github] <br>[![Open In Collab](../../assets/colab.svg)][matryoshka_colab]  |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Improve RAG with Re-ranking** 📈🔄 | Enhance your RAG applications by implementing **re-ranking strategies** for more relevant document retrieval. 📚🔍 | [![Github](../../assets/github.svg)][rag_reranking_github] <br>[![Open In Collab](../../assets/colab.svg)][rag_reranking_colab] <br>[![Ghost](../../assets/ghost.svg)][rag_reranking_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Instruct-Multitask** 🧠🎯 |  Integrate the **Instruct Embedding Model** with LanceDB to streamline your embedding API, reducing redundant code and overhead. 🌐📊 | [![Github](../../assets/github.svg)][instruct_multitask_github] <br>[![Open In Collab](../../assets/colab.svg)][instruct_multitask_colab] <br>[![Python](../../assets/python.svg)][instruct_multitask_python] <br>[![Ghost](../../assets/ghost.svg)][instruct_multitask_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Improve RAG with HyDE** 🌌🔍 | Use **Hypothetical Document Embeddings** for efficient, accurate, and unsupervised dense retrieval. 📄🔍 | [![Github](../../assets/github.svg)][hyde_github] <br>[![Open In Collab](../../assets/colab.svg)][hyde_colab]<br>[![Ghost](../../assets/ghost.svg)][hyde_ghost]                 |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Improve RAG with LOTR** 🧙‍♂️📜 | Enhance RAG with **Lord of the Retriever (LOTR)** to address 'Lost in the Middle' challenges, especially in medical data. 🌟📜 | [![Github](../../assets/github.svg)][lotr_github] <br>[![Open In Collab](../../assets/colab.svg)][lotr_colab] <br>[![Ghost](../../assets/ghost.svg)][lotr_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Advanced RAG: Parent Document Retriever** 📑🔗 | Use **Parent Document & Bigger Chunk Retriever** to maintain context and relevance when generating related content. 🎵📄 | [![Github](../../assets/github.svg)][parent_doc_retriever_github] <br>[![Open In Collab](../../assets/colab.svg)][parent_doc_retriever_colab] <br>[![Ghost](../../assets/ghost.svg)][parent_doc_retriever_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Corrective RAG with Langgraph** 🔧📊 | Enhance RAG reliability with **Corrective RAG (CRAG)** by self-reflecting and fact-checking for accurate and trustworthy results. ✅🔍 |[![Github](../../assets/github.svg)][corrective_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][corrective_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][corrective_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Contextual Compression with RAG** 🗜️🧠 | Apply **contextual compression techniques** to condense large documents while retaining essential information. 📄🗜️                                               | [![Github](../../assets/github.svg)][compression_rag_github] <br>[![Open In Collab](../../assets/colab.svg)][compression_rag_colab] <br>[![Ghost](../../assets/ghost.svg)][compression_rag_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Improve RAG with FLARE** 🔥|  Enable users to ask questions directly to academic papers, focusing on ArXiv papers, with Forward-Looking Active REtrieval augmented generation.🚀🌟                                                                       | [![Github](../../assets/github.svg)][flare_github] <br>[![Open In Collab](../../assets/colab.svg)][flare_colab] <br>[![Ghost](../../assets/ghost.svg)][flare_ghost] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Query Expansion and Reranker** 🔍🔄 | Enhance RAG with query expansion using Large Language Models and advanced **reranking methods** like Cross Encoders, ColBERT v2, and FlashRank for improved document retrieval precision and recall 🔍📈                                                         | [![Github](../../assets/github.svg)][query_github] <br>[![Open In Collab](../../assets/colab.svg)][query_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **RAG Fusion** ⚡🌐 | Revolutionize search with RAG Fusion, utilizing the **RRF algorithm** to rerank documents based on user queries, and leveraging LanceDB and OPENAI Embeddings for efficient information retrieval ⚡🌐                                                                          | [![Github](../../assets/github.svg)][fusion_github] <br>[![Open In Collab](../../assets/colab.svg)][fusion_colab] |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+| **Agentic RAG** 🤖📚 | Unlock autonomous information retrieval with **Agentic RAG**, a framework of **intelligent agents** that collaborate to synthesize, summarize, and compare data across sources, enabling proactive and informed decision-making 🤖📚                                                              | [![Github](../../assets/github.svg)][agentic_github] <br>[![Open In Collab](../../assets/colab.svg)][agentic_colab] |
+
+
+
+
+
+
+
+
+
+
+
+
+[matryoshka_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex
+[matryoshka_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/RAG-with_MatryoshkaEmbed-Llamaindex/RAG_with_MatryoshkaEmbedding_and_Llamaindex.ipynb
+
+[rag_reranking_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking
+[rag_reranking_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Reranking/main.ipynb
+[rag_reranking_ghost]: https://blog.lancedb.com/simplest-method-to-improve-rag-pipeline-re-ranking-cf6eaec6d544
+
+
+[instruct_multitask_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask
+[instruct_multitask_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.ipynb
+[instruct_multitask_python]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/instruct-multitask/main.py
+[instruct_multitask_ghost]: https://blog.lancedb.com/multitask-embedding-with-lancedb-be18ec397543
+
+[hyde_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE
+[hyde_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance-RAG-with-HyDE/main.ipynb
+[hyde_ghost]: https://blog.lancedb.com/advanced-rag-precise-zero-shot-dense-retrieval-with-hyde-0946c54dfdcb
+
+[lotr_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR
+[lotr_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Advance_RAG_LOTR/main.ipynb
+[lotr_ghost]: https://blog.lancedb.com/better-rag-with-lotr-lord-of-retriever-23c8336b9a35
+
+[parent_doc_retriever_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever
+[parent_doc_retriever_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/parent_document_retriever/main.ipynb
+[parent_doc_retriever_ghost]: https://blog.lancedb.com/modified-rag-parent-document-bigger-chunk-retriever-62b3d1e79bc6
+
+[corrective_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph
+[corrective_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Corrective-RAG-with_Langgraph/CRAG_with_Langgraph.ipynb
+[corrective_rag_ghost]: https://blog.lancedb.com/implementing-corrective-rag-in-the-easiest-way-2/
+
+[compression_rag_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG
+[compression_rag_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Contextual-Compression-with-RAG/main.ipynb
+[compression_rag_ghost]: https://blog.lancedb.com/enhance-rag-integrate-contextual-compression-and-filtering-for-precision-a29d4a810301/
+
+[flare_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR
+[flare_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/better-rag-FLAIR/main.ipynb
+[flare_ghost]: https://blog.lancedb.com/better-rag-with-active-retrieval-augmented-generation-flare-3b66646e2a9f/
+
+[query_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker
+[query_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/QueryExpansion&Reranker/main.ipynb
+
+
+[fusion_github]: https://github.com/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion
+[fusion_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/RAG_Fusion/main.ipynb
+
+[agentic_github]: https://github.com/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG
+[agentic_colab]: https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/tutorials/Agentic_RAG/main.ipynb
+
+

docs/src/fts.md

@@ -2,7 +2,6 @@
 
 LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
-A hybrid search solution combining vector and full-text search is also on the way.
 
 ## Installation
 
@@ -55,6 +54,16 @@ This returns the result as a list of dictionaries as follows.
 !!! note
     LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("text", tokenizer_name="en_stem")
+```
+
+The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+
 ## Index multiple columns
 
 If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
@@ -140,6 +149,7 @@ is treated as a phrase query.
 In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
 double quotes replaced by single quotes.
 
+
 ## Configurations
 
 By default, LanceDB configures a 1GB heap size limit for creating the index. You can

docs/src/guides/storage.md

@@ -32,41 +32,232 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
     db = lancedb.connect("az://bucket/path")
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    AWS S3:
+    === "@lancedb/lancedb"
 
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path");
+        AWS S3:
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
+
+        Google Cloud Storage:
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
+
+        Azure Blob Storage:
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("az://bucket/path");
+        ```
+
+
+    === "vectordb (deprecated)"
+
+        AWS S3:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
+
+        Google Cloud Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
+
+        Azure Blob Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("az://bucket/path");
+        ```
+
+In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
+
+```bash
+export TIMEOUT=60s
+```
+
+!!! note "`storage_options` availability"
+
+    The `storage_options` parameter is only available in Python *async* API and JavaScript API.
+    It is not yet supported in the Python synchronous API.
+
+If you only want this to apply to one particular connection, you can pass the `storage_options` argument when opening the connection:
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={"timeout": "60s"}
+    )
     ```
 
-    Google Cloud Storage:
+=== "TypeScript"
 
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("gs://bucket/path");
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
+
+Getting even more specific, you can set the `timeout` for only a particular table:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async("s3://bucket/path")
+    table = await db.create_table(
+        "table",
+        [{"a": 1, "b": 2}],
+        storage_options={"timeout": "60s"}
+    )
     ```
 
-    Azure Blob Storage:
+=== "TypeScript"
 
-    ```javascript
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("az://bucket/path");
-    ```
+    === "@lancedb/lancedb"
 
-In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided using environment variables. In general, these environment variables are the same as those used by the respective cloud SDKs. The sections below describe the environment variables that can be used to configure each object store.
+        <!-- skip-test -->
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("s3://bucket/path");
+        const table = db.createTable(
+            "table",
+            [{ a: 1, b: 2}],
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
 
-LanceDB OSS uses the [object-store](https://docs.rs/object_store/latest/object_store/) Rust crate for object store access. There are general environment variables that can be used to configure the object store, such as the request timeout and proxy configuration. See the [object_store ClientConfigKey](https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html) doc for available configuration options. The environment variables that can be set are the snake-cased versions of these variable names. For example, to set `ProxyUrl` use the environment variable `PROXY_URL`. (Don't let the Rust docs intimidate you! We link to them so you can see an up-to-date list of the available options.)
+    === "vectordb (deprecated)"
 
+        <!-- skip-test -->
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        const table = db.createTable(
+            "table",
+            [{ a: 1, b: 2}],
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
+
+!!! info "Storage option casing"
+
+    The storage option keys are case-insensitive. So `connect_timeout` and `CONNECT_TIMEOUT` are the same setting. Usually lowercase is used in the `storage_options` argument and uppercase is used for environment variables. In the `lancedb` Node package, the keys can also be provided in `camelCase` capitalization. For example, `connectTimeout` is equivalent to `connect_timeout`.
+
+### General configuration
+
+There are several options that can be set for all object stores, mostly related to network client configuration.
+
+<!-- from here: https://docs.rs/object_store/latest/object_store/enum.ClientConfigKey.html -->
+
+| Key                        | Description                                                                                      |
+|----------------------------|--------------------------------------------------------------------------------------------------|
+| `allow_http`               | Allow non-TLS, i.e. non-HTTPS connections. Default: `False`.                                      |
+| `allow_invalid_certificates`| Skip certificate validation on HTTPS connections. Default: `False`.                               |
+| `connect_timeout`          | Timeout for only the connect phase of a Client. Default: `5s`.                                    |
+| `timeout`                  | Timeout for the entire request, from connection until the response body has finished. Default: `30s`. |
+| `user_agent`               | User agent string to use in requests.                                                             |
+| `proxy_url`                | URL of a proxy server to use for requests. Default: `None`.                                       |
+| `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
+| `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
 
 ### AWS S3
 
-To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables.
+To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
+These can be set as environment variables or passed in the `storage_options` parameter:
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "aws_access_key_id": "my-access-key",
+            "aws_secret_access_key": "my-secret-key",
+            "aws_session_token": "my-session-token",
+        }
+    )
+    ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
+            }
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
+            }
+        );
+        ```
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
-You can see a full list of environment variables [here](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html#method.from_env).
+The following keys can be used as both environment variables or keys in the `storage_options` parameter:
+
+| Key                                | Description                                                                                          |
+|------------------------------------|------------------------------------------------------------------------------------------------------|
+| `aws_region` / `region`             | The AWS region the bucket is in. This can be automatically detected when using AWS S3, but must be specified for S3-compatible stores. |
+| `aws_access_key_id` / `access_key_id` | The AWS access key ID to use.                                                                       |
+| `aws_secret_access_key` / `secret_access_key` | The AWS secret access key to use.                                                               |
+| `aws_session_token` / `session_token` | The AWS session token to use.                                                                     |
+| `aws_endpoint` / `endpoint`         | The endpoint to use for S3-compatible stores.                                                       |
+| `aws_virtual_hosted_style_request` / `virtual_hosted_style_request` | Whether to use virtual hosted-style requests, where the bucket name is part of the endpoint. Meant to be used with `aws_endpoint`. Default: `False`. |
+| `aws_s3_express` / `s3_express`     | Whether to use S3 Express One Zone endpoints. Default: `False`. See more details below.             |
+| `aws_server_side_encryption`        | The server-side encryption algorithm to use. Must be one of `"AES256"`, `"aws:kms"`, or `"aws:kms:dsse"`. Default: `None`. |
+| `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
+| `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
 
 !!! tip "Automatic cleanup for failed writes"
 
@@ -144,24 +335,349 @@ For **read-only access**, LanceDB will need a policy such as:
 }
 ```
 
+#### DynamoDB Commit Store for concurrent writes
+
+By default, S3 does not support concurrent writes. Having two or more processes
+writing to the same table at the same time can lead to data corruption. This is
+because S3, unlike other object stores, does not have any atomic put or copy
+operation.
+
+To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
+as a commit store. This table will be used to coordinate writes between
+different processes. To enable this feature, you must modify your connection
+URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
+name of the table to use.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+
+    const db = await lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    );
+    ```
+
+The DynamoDB table must be created with the following schema:
+
+- Hash key: `base_uri` (string)
+- Range key: `version` (number)
+
+You can create this programmatically with:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import boto3
+
+    dynamodb = boto3.client("dynamodb")
+    table = dynamodb.create_table(
+        TableName=table_name,
+        KeySchema=[
+            {"AttributeName": "base_uri", "KeyType": "HASH"},
+            {"AttributeName": "version", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "base_uri", "AttributeType": "S"},
+            {"AttributeName": "version", "AttributeType": "N"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+    )
+    ```
+
+=== "JavaScript"
+
+    <!-- skip-test -->
+    ```javascript
+    import {
+      CreateTableCommand,
+      DynamoDBClient,
+    } from "@aws-sdk/client-dynamodb";
+
+    const dynamodb = new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+    const command = new CreateTableCommand({
+      TableName: table_name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    ```
+
+
 #### S3-compatible stores
 
-LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify two environment variables: `AWS_ENDPOINT` and `AWS_DEFAULT_REGION`. `AWS_ENDPOINT` should be the URL of the S3-compatible store, and `AWS_DEFAULT_REGION` should be the region to use.
+LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
 
-<!-- TODO: we should also document the use of S3 Express once we fully support it -->
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://bucket/path",
+        storage_options={
+            "region": "us-east-1",
+            "endpoint": "http://minio:9000",
+        }
+    )
+    ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
+            }
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
+            }
+        );
+        ```
+
+This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
+
+!!! tip "Local servers"
+
+    For local development, the server often has a `http` endpoint rather than a
+    secure `https` endpoint. In this case, you must also set the `ALLOW_HTTP`
+    environment variable to `true` to allow non-TLS connections, or pass the
+    storage option `allow_http` as `true`. If you do not do this, you will get
+    an error like `URL scheme is not allowed`.
+
+#### S3 Express
+
+LanceDB supports [S3 Express One Zone](https://aws.amazon.com/s3/storage-classes/express-one-zone/) endpoints, but requires additional configuration. Also, S3 Express endpoints only support connecting from an EC2 instance within the same region.
+
+To configure LanceDB to use an S3 Express endpoint, you must set the storage option `s3_express`. The bucket name in your table URI should **include the suffix**.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3://my-bucket--use1-az4--x-s3/path",
+        storage_options={
+            "region": "us-east-1",
+            "s3_express": "true",
+        }
+    )
+    ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
+            }
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
+            }
+        );
+        ```
 
 ### Google Cloud Storage
 
-GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. There are several aliases for this environment variable, documented [here](https://docs.rs/object_store/latest/object_store/gcp/struct.GoogleCloudStorageBuilder.html#method.from_env).
+GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environment variable to the path of a JSON file containing the service account credentials. Alternatively, you can pass the path to the JSON file in the `storage_options`:
 
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "gs://my-bucket/my-database",
+        storage_options={
+            "service_account": "path/to/service-account.json",
+        }
+    )
+    ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect(
+            "gs://my-bucket/my-database",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
+            }
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "gs://my-bucket/my-database",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
+            }
+        );
+        ```
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
+The following keys can be used as both environment variables or keys in the `storage_options` parameter:
+<!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
+
+| Key                                   | Description                                  |
+|---------------------------------------|----------------------------------------------|
+| ``google_service_account`` / `service_account` | Path to the service account JSON file.       |
+| ``google_service_account_key``        | The serialized service account key.          |
+| ``google_application_credentials``    | Path to the application credentials.         |
+
 ### Azure Blob Storage
 
-Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME` and ``AZURE_STORAGE_ACCOUNT_KEY`` environment variables. The full list of environment variables that can be set are documented [here](https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html#method.from_env).
+Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
 
+=== "Python"
 
-<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->
+    <!-- skip-test -->
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "az://my-container/my-database",
+        storage_options={
+            account_name: "some-account",
+            account_key: "some-key",
+        }
+    )
+    ```
+
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect(
+            "az://my-container/my-database",
+            {
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
+            }
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "az://my-container/my-database",
+            {
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
+            }
+        );
+        ```
+
+These keys can be used as both environment variables or keys in the `storage_options` parameter:
+
+<!-- source: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html -->
+
+| Key                                   | Description                                                                                      |
+|---------------------------------------|--------------------------------------------------------------------------------------------------|
+| ``azure_storage_account_name``        | The name of the azure storage account.                                                           |
+| ``azure_storage_account_key``         | The serialized service account key.                                                              |
+| ``azure_client_id``                   | Service principal client id for authorizing requests.                                            |
+| ``azure_client_secret``               | Service principal client secret for authorizing requests.                                        |
+| ``azure_tenant_id``                   | Tenant id used in oauth flows.                                                                   |
+| ``azure_storage_sas_key``             | Shared access signature. The signature is expected to be percent-encoded, much like they are provided in the azure storage explorer or azure portal. |
+| ``azure_storage_token``               | Bearer token.                                                                                    |
+| ``azure_storage_use_emulator``        | Use object store with azurite storage emulator.                                                  |
+| ``azure_endpoint``                    | Override the endpoint used to communicate with blob storage.                                      |
+| ``azure_use_fabric_endpoint``         | Use object store with url scheme account.dfs.fabric.microsoft.com.                               |
+| ``azure_msi_endpoint``                | Endpoint to request a imds managed identity token.                                               |
+| ``azure_object_id``                   | Object id for use with managed identity authentication.                                          |
+| ``azure_msi_resource_id``             | Msi resource id for use with managed identity authentication.                                    |
+| ``azure_federated_token_file``        | File containing token for Azure AD workload identity federation.                                 |
+| ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
+| ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
+
+<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables.md

@@ -3,32 +3,46 @@
 
 A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
 
-This guide will show how to create tables, insert data into them, and update the data.  
+This guide will show how to create tables, insert data into them, and update the data.
 
 
 ## Creating a LanceDB Table
 
+Initialize a LanceDB connection and create a table
+
 === "Python"
-    Initialize a LanceDB connection and create a table using one of the many methods listed below.
 
     ```python
     import lancedb
     db = lancedb.connect("./.lancedb")
     ```
 
-=== "Javascript"
-
-    Initialize a VectorDB connection and create a table using one of the many methods listed below.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const lancedb = require("vectordb");
+        const arrow = require("apache-arrow");
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+
+
 ### From list of tuples or dictionaries
 
 === "Python"
@@ -45,102 +59,150 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
+
     !!! info "Note"
-        If the table already exists, LanceDB will raise an error by default. 
+        If the table already exists, LanceDB will raise an error by default.
 
         `create_table` supports an optional `exist_ok` parameter. When set to True
         and the table exists, then it simply opens the existing table. The data you
         passed in will NOT be appended to the table in that case.
 
-        ```python
-        db.create_table("name", data, exist_ok=True)
-        ```
-        
-        Sometimes you want to make sure that you start fresh. If you want to 
-        overwrite the table, you can pass in mode="overwrite" to the createTable function.
-
-        ```python
-        db.create_table("name", data, mode="overwrite")
-        ```
-
-=== "Javascript"
-    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
-
-    ```javascript
-    const tb = await db.createTable("my_table", [{
-        "vector": [3.1, 4.1],
-        "item": "foo",
-        "price": 10.0
-    }, {
-        "vector": [5.9, 26.5],
-        "item": "bar",
-        "price": 20.0
-    }]);
-    ```
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```python
+    db.create_table("name", data, exist_ok=True)
     ```
 
-    ### From a Pandas DataFrame
+    Sometimes you want to make sure that you start fresh. If you want to
+    overwrite the table, you can pass in mode="overwrite" to the createTable function.
 
     ```python
-    import pandas as pd
-
-    data = pd.DataFrame({
-        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
-        "lat": [45.5, 40.1],
-        "long": [-122.7, -74.1]
-    })
-
-    db.create_table("my_table", data)
-
-    db["my_table"].head()
+    db.create_table("name", data, mode="overwrite")
     ```
-    !!! info "Note"
+
+=== "Typescript[^1]"
+    You can create a LanceDB table in JavaScript using an array of records as follows.
+
+    === "@lancedb/lancedb"
+
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table"
+        ```
+
+        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
+        ```
+
+        !!! info "Note"
+            `createTable` supports an optional `existsOk` parameter. When set to true
+            and the table exists, then it simply opens the existing table. The data you
+            passed in will NOT be appended to the table in that case.
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
+        ```
+
+        Sometimes you want to make sure that you start fresh. If you want to
+        overwrite the table, you can pass in mode: "overwrite" to the createTable function.
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/basic_legacy.ts:create_table"
+        ```
+
+        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use apache-arrow to declare a schema
+
+
+
+        ```ts
+        --8<-- "docs/src/basic_legacy.ts:create_table_with_schema"
+        ```
+
+        !!! warning
+            `existsOk` is not available in `vectordb`
+
+
+
+            If the table already exists, vectordb will raise an error by default.
+            You can use `writeMode: WriteMode.Overwrite` to overwrite the table.
+            But this will delete the existing table and create a new one with the same name.
+
+
+        Sometimes you want to make sure that you start fresh.
+
+        If you want to overwrite the table, you can pass in `writeMode: lancedb.WriteMode.Overwrite` to the createTable function.
+
+        ```ts
+        const table = await con.createTable(tableName, data, {
+            writeMode: WriteMode.Overwrite
+        })
+        ```
+
+### From a Pandas DataFrame
+
+```python
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+    "lat": [45.5, 40.1],
+    "long": [-122.7, -74.1]
+})
+
+db.create_table("my_table", data)
+
+db["my_table"].head()
+```
+
+!!! info "Note"
     Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
-    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
+The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.
 
-    ```python
-    custom_schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 4)),
-    pa.field("lat", pa.float32()),
-    pa.field("long", pa.float32())
-    ])
+```python
+custom_schema = pa.schema([
+pa.field("vector", pa.list_(pa.float32(), 4)),
+pa.field("lat", pa.float32()),
+pa.field("long", pa.float32())
+])
 
-    table = db.create_table("my_table", data, schema=custom_schema)
-    ```
+table = db.create_table("my_table", data, schema=custom_schema)
+```
 
 ### From a Polars DataFrame
 
-    LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
-    written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
-    under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
-    is on the way.
+LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
+written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
+under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
+is on the way.
 
-    ```python
-    import polars as pl
+```python
+import polars as pl
 
-    data = pl.DataFrame({
-        "vector": [[3.1, 4.1], [5.9, 26.5]],
-        "item": ["foo", "bar"],
-        "price": [10.0, 20.0]
-    })
-    table = db.create_table("pl_table", data=data)
-    ```
+data = pl.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pl_table", data=data)
+```
 
 ### From an Arrow Table
+You can also create LanceDB tables directly from Arrow tables.
+LanceDB supports float16 data type!
+
 === "Python"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports float16 data type!
 
     ```python
     import pyarrows as pa
     import numpy as np
-    
+
     dim = 16
     total = 2
     schema = pa.schema(
@@ -160,13 +222,19 @@ This guide will show how to create tables, insert data into them, and update the
     tbl = db.create_table("f16_tbl", data, schema=schema)
     ```
 
-=== "Javascript"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports Float16 data type!
+=== "Typescript[^1]"
 
-    ```javascript
-    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
-    ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_f16_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+        ```
 
 ### From Pydantic Models
 
@@ -225,7 +293,7 @@ class NestedSchema(LanceModel):
 tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
 ```
 
-This creates a struct column called "document" that has two subfields 
+This creates a struct column called "document" that has two subfields
 called "content" and "source":
 
 ```
@@ -236,7 +304,7 @@ vector: fixed_size_list<item: float>[1536] not null
     child 0, item: float
 document: struct<content: string not null, source: string not null> not null
     child 0, content: string not null
-    child 1, source: string not null    
+    child 1, source: string not null
 ```
 
 #### Validators
@@ -261,7 +329,7 @@ class TestModel(LanceModel):
     @classmethod
     def tz_must_match(cls, dt: datetime) -> datetime:
         assert dt.tzinfo == tz
-        return dt        
+        return dt
 
 ok = TestModel(dt_with_tz=datetime.now(tz))
 
@@ -329,23 +397,24 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
     tbl = db.open_table("my_table")
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
+
     If you forget the name of your table, you can always get a listing of all table names.
 
-    ```javascript
+    ```typescript
     console.log(await db.tableNames());
     ```
 
     Then, you can open any existing tables.
 
-    ```javascript
+    ```typescript
     const tbl = await db.openTable("my_table");
     ```
 
 ## Creating empty table
+You can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
 === "Python"
-    In Python, you can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
     ```python
 
@@ -364,8 +433,8 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
     tbl = db.create_table("empty_table_add", schema=schema)
     ```
 
-    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not 
-    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` 
+    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
+    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
     that has been extended to support LanceDB specific types like `Vector`.
 
     ```python
@@ -382,9 +451,23 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 
     Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
+
 ## Adding to a table
 
-After a table has been created, you can always add more data to it using the various methods available.
+After a table has been created, you can always add more data to it usind the `add` method
 
 === "Python"
     You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
@@ -452,8 +535,27 @@ After a table has been created, you can always add more data to it using the var
     tbl.add(pydantic_model_items)
     ```
 
+    ??? "Ingesting Pydantic models with LanceDB embedding API"
+        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
 
-=== "JavaScript"
+        ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import get_registry
+
+        db = lancedb.connect("~/tmp")
+        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
+
+        class Schema(LanceModel):
+            text: str = embed_fcn.SourceField()
+            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
+
+        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
+        models = [Schema(text="hello"), Schema(text="world")]
+        tbl.add(models)
+        ```
+
+=== "Typescript[^1]"
 
     ```javascript
     await tbl.add(
@@ -509,15 +611,15 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     # 0  3  [5.0, 6.0]
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
 
-    ```javascript
+    ```ts
     await tbl.delete('item = "fizz"')
     ```
 
     ### Deleting row with specific column value
 
-    ```javascript
+    ```ts
     const con = await lancedb.connect("./.lancedb")
     const data = [
       {id: 1, vector: [1, 2]},
@@ -531,7 +633,7 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
     ### Delete from a list of values
 
-    ```javascript
+    ```ts
     const to_remove = [1, 5];
     await tbl.delete(`id IN (${to_remove.join(",")})`)
     await tbl.countRows() // Returns 1
@@ -588,26 +690,49 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+    === "@lancedb/lancedb"
 
-    ```javascript
-    const lancedb = require("vectordb");
+        API Reference: [lancedb.Table.update](../js/classes/Table.md/#update)
 
-    const db = await lancedb.connect("./.lancedb");
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
 
-    const data = [
-      {x: 1, vector: [1, 2]},
-      {x: 2, vector: [3, 4]},
-      {x: 3, vector: [5, 6]},
-    ];
-    const tbl = await db.createTable("my_table", data)
+        const db = await lancedb.connect("./.lancedb");
 
-    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
-    ```
+        const data = [
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
 
-The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
+        await tbl.update({vector: [10, 10]}, { where: "x = 2"})
+        ```
+
+    === "vectordb (deprecated)"
+
+        API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+
+        ```ts
+        const lancedb = require("vectordb");
+
+        const db = await lancedb.connect("./.lancedb");
+
+        const data = [
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
+
+        await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        ```
+
+#### Updating using a sql query
+
+  The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
 
 === "Python"
 
@@ -626,16 +751,47 @@ The `values` parameter is used to provide the new values for the columns as lite
     2  3  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    ```javascript
-    await tbl.update({ valuesSql: { x: "x + 1" } })
-    ```
+    === "@lancedb/lancedb"
+
+        Coming Soon!
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        await tbl.update({ valuesSql: { x: "x + 1" } })
+        ```
 
 !!! info "Note"
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## Drop a table
+
+Use the `drop_table()` method on the database to remove a table.
+
+=== "Python"
+
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
+
+=== "TypeScript"
+
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised.
+
+
 ## Consistency
 
 In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
@@ -651,7 +807,7 @@ There are three possible settings for `read_consistency_interval`:
     This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
 
 === "Python"
-    
+
     To set strong consistency, use `timedelta(0)`:
 
     ```python
@@ -673,33 +829,35 @@ There are three possible settings for `read_consistency_interval`:
     ```python
     db = lancedb.connect("./.lancedb")
     table = db.open_table("my_table")
-    
+
     # (Other writes happen to my_table from another process)
 
     # Check for updates
     table.checkout_latest()
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
     To set strong consistency, use `0`:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
     const table = await db.openTable("my_table");
     ```
 
     For eventual consistency, specify the update interval as seconds:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
     const table = await db.openTable("my_table");
     ```
 
-<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
+<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007
     Once it does, we can show manual consistency check for Node as well.
 -->
 
 ## What's next?
 
-Learn the best practices on creating an ANN index and getting the most out of it.
+Learn the best practices on creating an ANN index and getting the most out of it.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -0,0 +1,131 @@
+## Improving retriever performance
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+
+There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
+
+* Using different query types
+* Using hybrid search
+* Fine-tuning the embedding models
+* Using different embedding models
+
+Using different embedding models is something that's very specific to the use case and the data. So we will not discuss it here. In this section, we will discuss the first three techniques.
+
+
+!!! note "Note"
+    We'll be using a simple metric called "hit-rate" for evaluating the performance of the retriever across this guide. Hit-rate is the percentage of queries for which the retriever returned the correct answer in the top-k results. For example, if the retriever returned the correct answer in the top-3 results for 70% of the queries, then the hit-rate@3 is 0.7.
+
+
+## The dataset
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
+
+### Using different query types
+Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
+
+```python
+import lancedb
+import pandas as pd
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import Vector, LanceModel
+
+db = lancedb.connect("~/lancedb/query_types")
+df = pd.read_csv("data_qa.csv")
+
+embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.")
+
+class Schema(LanceModel):
+    context: str = embed_fcn.SourceField()
+    vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField()
+
+table = db.create_table("qa", schema=Schema)
+table.add(df[["context"]].to_dict(orient="records"))
+
+queries = df["query"].tolist()
+```
+
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
+
+* <b> Vector Search: </b>
+
+    ```python
+    table.search(quries[0], query_type="vector").limit(5).to_pandas()
+    ```
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
+
+    ```python
+    table.search(quries[0]).limit(5).to_pandas()
+    ```
+
+    Vector or semantic search is useful when you want to find documents that are similar to the query in terms of meaning.
+
+---
+
+* <b> Full-text Search: </b>
+    
+    FTS requires creating an index on the column you want to search on. `replace=True` will replace the existing index if it exists.
+    Once the index is created, you can search using the `fts` query type.
+    ```python
+    table.create_fts_index("context", replace=True)
+    table.search(quries[0], query_type="fts").limit(5).to_pandas()
+    ```
+
+    Full-text search is useful when you want to find documents that contain the query terms.
+
+---
+
+* <b> Hybrid Search: </b>
+
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
+    ```python
+    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
+    ```
+    Hybrid search requires a reranker to combine and rank the results from vector and full-text search. We'll cover reranking as a concept in the next section.
+
+    Hybrid search is useful when you want to combine the benefits of both vector and full-text search.
+
+    !!! note "Note"
+        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
+
+    
+
+### Hit rate evaluation results
+
+Now that we have seen how to run different query types on the dataset, let's evaluate the hit-rate of each query type on the dataset.
+For brevity, the entire evaluation script is not shown here. You can find the complete evaluation and benchmarking utility scripts [here](https://github.com/lancedb/ragged).
+
+Here are the hit-rate results for the dataset:
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Hybrid Search (w/ LinearCombinationReranker) | 0.645 |
+
+**Choosing query type** is very specific to the use case and the data. This synthetic dataset has been generated to be semantically challenging, i.e, the queries don't have a lot of keywords in common with the context. So, vector search performs better than full-text search. However, in real-world scenarios, full-text search might perform better than vector search. Hybrid search is a good choice when you want to combine the benefits of both vector and full-text search.
+
+### Evaluation results on other datasets
+
+The hit-rate results can vary based on the dataset and the query type. Here are the hit-rate results for the other datasets using the same embedding function.
+
+* <b> SQuAD Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.822 |
+    | Full-text Search | 0.835 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.8874 |
+
+* <b> Uber10K sec filing Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.608 |
+    | Full-text Search | 0.82 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.80 |
+
+In these standard datasets, FTS seems to perform much better than vector search because the queries have a lot of keywords in common with the context. So, in general choosing the query type is very specific to the use case and the data.
+
+

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -0,0 +1,80 @@
+Continuing from the previous section, we can now rerank the results using more complex rerankers.
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+## Reranking search results
+You can rerank any search results using a reranker. The syntax for reranking is as follows:
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker()
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+```
+Based on the `query_type`, the `rerank()` function can accept other arguments as well. For example, hybrid search accepts a `normalize` param to determine the score normalization method.
+
+!!! note "Note"
+    LanceDB provides a `Reranker` base class that can be extended to implement custom rerankers. Each reranker must implement the `rerank_hybrid` method. `rerank_vector` and `rerank_fts` methods are optional. For example, the `LinearCombinationReranker` only implements the `rerank_hybrid` method and so it can only be used for reranking hybrid search results.
+
+## Choosing a Reranker
+There are many rerankers available in LanceDB like `CrossEncoderReranker`, `CohereReranker`, and `ColBERT`. The choice of reranker depends on the dataset and the application. You can even implement you own custom reranker by extending the `Reranker` class. For more details about each available reranker and performance comparison, refer to the [rerankers](https://lancedb.github.io/lancedb/reranking/) documentation.
+
+In this example, we'll use the `CohereReranker` to rerank the search results. It requires  `cohere` to be installed and `COHERE_API_KEY` to be set in the environment. To get your API key, sign up on [Cohere](https://cohere.ai/).
+
+```python
+from lancedb.rerankers import CohereReranker
+
+# use Cohere reranker v3
+reranker = CohereReranker(model_name="rerank-english-v3.0") # default model is "rerank-english-v2.0"
+```
+
+### Reranking search results
+Now we can rerank all query type results using the `CohereReranker`:
+
+```python
+
+# rerank hybrid search results
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank vector search results
+table.search(quries[0], query_type="vector").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank fts search results
+table.search(quries[0], query_type="fts").rerank(reranker=reranker).limit(5).to_pandas()
+```
+
+Each reranker can accept additional arguments. For example, `CohereReranker` accepts `top_k` and `batch_size` params to control the number of documents to rerank and the batch size for reranking respectively. Similarly, a custom reranker can accept any number of arguments based on the implementation. For example, a reranker can accept a `filter` that implements some custom logic to filter out documents before reranking.
+
+## Results
+
+Let us take a look at the same datasets from the previous sections, using the same embedding table but with Cohere reranker applied to all query types.
+
+!!! note "Note"
+    When reranking fts or vector search results, the search results are over-fetched by a factor of 2 and then reranked. From the reranked set, `top_k` (5 in this case) results are taken. This is done because reranking will have no effect on the hit-rate if we only fetch the `top_k` results.
+
+### Synthetic LLama2 paper dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.640 |
+| FTS   |  0.595  |
+| Reranked vector | 0.677    |
+| Reranked fts  | 0.672    |
+| Hybrid | 0.759 |
+
+### SQuAD Dataset
+
+
+### Uber10K sec filing Dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.608 |
+| FTS   |  0.824  |
+| Reranked vector | 0.671    |
+| Reranked fts  | 0.843    |
+| Hybrid | 0.849 |
+
+
+
+

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -0,0 +1,82 @@
+## Finetuning the Embedding Model
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
+
+We'll use the same dataset as in the previous sections. Start off by splitting the dataset into training and validation sets:
+```python
+from sklearn.model_selection import train_test_split
+
+train_df, validation_df = train_test_split("data_qa.csv", test_size=0.2, random_state=42)
+
+train_df.to_csv("data_train.csv", index=False)
+validation_df.to_csv("data_val.csv", index=False)
+```
+
+You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
+
+
+Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
+```python
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.readers.file import PagedCSVReader
+from llama_index.finetuning import generate_qa_embedding_pairs
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+
+def load_corpus(file):
+    loader = PagedCSVReader(encoding="utf-8")
+    docs = loader.load_data(file=Path(file))
+
+    parser = SentenceSplitter()
+    nodes = parser.get_nodes_from_documents(docs)
+
+    return nodes
+
+from llama_index.llms.openai import OpenAI
+
+
+train_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=train_nodes, verbose=False
+)
+val_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=val_nodes, verbose=False
+)
+```
+
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
+
+```python
+from llama_index.finetuning import SentenceTransformersFinetuneEngine
+
+finetune_engine = SentenceTransformersFinetuneEngine(
+    train_dataset,
+    model_id="BAAI/bge-small-en-v1.5",
+    model_output_path="tuned_model",
+    val_dataset=val_dataset,
+)
+finetune_engine.finetune()
+embed_model = finetune_engine.get_finetuned_model()
+```
+This saves the fine tuned embedding model in `tuned_model` folder. This al
+
+# Evaluation results
+In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 
+On performing the same hit-rate evaluation as before, we see a significant improvement in the hit-rate across all query types.
+
+### Baseline
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.677 |
+| Reranked Full-text Search | 0.672 |
+| Hybrid Search (w/ CohereReranker) | 0.759|
+
+### Fine-tuned model ( 2 iterations )
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.672 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.754 |
+| Reranked Full-text Search | 0.672|
+| Hybrid Search (w/ CohereReranker) | 0.768 |

docs/src/hybrid_search/eval.md

@@ -5,7 +5,9 @@ Hybrid Search is a broad (often misused) term. It can mean anything from combini
 ## The challenge of (re)ranking search results
 Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.
+
 * <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+
 * <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
 
 Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.

docs/src/hybrid_search/hybrid_search.md

@@ -5,9 +5,6 @@ LanceDB supports both semantic and keyword-based search (also termed full-text s
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
 
-!!! note
-    You need to create a full-text search index before performing a hybrid search. You can create a full-text search index using the `create_fts_index()` method of the table object.
-
 ```python
 import os
 
@@ -58,7 +55,188 @@ By default, LanceDB uses `LinearCombinationReranker(weight=0.7)` to combine and
 
 
 ## Available Rerankers
-LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Visit the [rerankers](../reranking/) page for more information on each re-ranker.
+LanceDB provides a number of re-rankers out of the box. You can use any of these re-rankers by passing them to the `rerank()` method. Here's a list of available re-rankers:
 
-## Custom Rerankers
-You can also create custom rerankers by extending the base `Reranker` class. The custom reranker should implement the `rerank` method that takes a list of search results and returns a reranked list of search results. Visit the [custom rerankers](../reranking/custom_reranker.md) page for more information on creating custom rerankers.
+### Linear Combination Reranker
+This is the default re-ranker used by LanceDB. It combines the results of semantic and full-text search using a linear combination of the scores. The weights for the linear combination can be specified. It defaults to 0.7, i.e, 70% weight for semantic search and 30% weight for full-text search.
+
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vector search
+
+results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `weight`: `float`, default `0.7`:
+    The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
+* `fill`: `float`, default `1.0`:
+        The score to give to results that are only in one of the two result sets.This is treated as penalty, so a higher value means a lower score.
+        TODO: We should just hardcode this-- its pretty confusing as we invert scores to calculate final score
+* `return_score` : str, default `"relevance"`
+        options are "relevance" or "all"
+        The type of score to return. If "relevance", will return only the `_relevance_score. If "all", will return all scores from the vector and FTS search along with the relevance score.
+
+### Cohere Reranker
+This re-ranker uses the [Cohere](https://cohere.ai/) API to combine the results of semantic and full-text search. You can use this re-ranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll need to set the `COHERE_API_KEY` environment variable to use this re-ranker.
+
+```python
+from lancedb.rerankers import CohereReranker
+
+reranker = CohereReranker()
+
+results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : str, default `"rerank-english-v2.0"`
+        The name of the cross encoder model to use. Available cohere models are:
+        - rerank-english-v2.0
+        - rerank-multilingual-v2.0
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `top_n` : str, default `None`
+        The number of results to return. If None, will return all results.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### Cross Encoder Reranker
+This reranker uses the [Sentence Transformers](https://www.sbert.net/) library to combine the results of semantic and full-text search. You can use it by passing `CrossEncoderReranker()` to the `rerank()` method.
+
+```python
+from lancedb.rerankers import CrossEncoderReranker
+
+reranker = CrossEncoderReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+
+### Arguments
+----------------
+* `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
+        The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
+* `column` : str, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `device` : str, default `None`
+        The device to use for the cross encoder model. If None, will use "cuda" if available, otherwise "cpu".
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+
+### ColBERT Reranker
+This reranker uses the ColBERT model to combine the results of semantic and full-text search. You can use it by passing `ColbertrReranker()` to the `rerank()` method. 
+
+ColBERT reranker model calculates relevance of given docs against the query and don't take existing fts and vector search scores into account, so it currently only supports `return_score="relevance"`. By default, it looks for `text` column to rerank the results. But you can specify the column name to use as input to the cross encoder model as described below.
+
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
+        The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+        The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+        options are `"relevance"` or `"all"`. Only `"relevance"` is supported for now.
+
+!!! Note
+    Only returns `_relevance_score`. Does not support `return_score = "all"`.
+
+### OpenAI Reranker
+This reranker uses the OpenAI API to combine the results of semantic and full-text search. You can use it by passing `OpenaiReranker()` to the `rerank()` method.
+
+!!! Note
+    This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
+
+!!! Tip
+    - You might run out of token limit so set the search `limits` based on your token limit.
+    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
+
+```python
+from lancedb.rerankers import OpenaiReranker
+
+reranker = OpenaiReranker()
+
+results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
+```
+
+### Arguments
+----------------
+* `model_name` : `str`, default `"gpt-4-turbo-preview"`
+    The name of the cross encoder model to use.
+* `column` : `str`, default `"text"`
+    The name of the column to use as input to the cross encoder model.
+* `return_score` : `str`, default `"relevance"`
+    options are "relevance" or "all". Only "relevance" is supported for now.
+* `api_key` : `str`, default `None`
+    The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+
+
+## Building Custom Rerankers
+You can build your own custom reranker by subclassing the `Reranker` class and implementing the `rerank_hybrid()` method. Here's an example of a custom reranker that combines the results of semantic and full-text search using a linear combination of the scores.
+
+The `Reranker` base interface comes with a `merge_results()` method that can be used to combine the results of semantic and full-text search. This is a vanilla merging algorithm that simply concatenates the results and removes the duplicates without taking the scores into consideration. It only keeps the first copy of the row encountered. This works well in cases that don't require the scores of semantic and full-text search to combine the results. If you want to use the scores or want to support `return_score="all"`, you'll need to implement your own merging algorithm.
+
+```python
+
+from lancedb.rerankers import Reranker
+import pyarrow as pa
+
+class MyReranker(Reranker):
+    def __init__(self, param1, param2, ..., return_score="relevance"):
+        super().__init__(return_score)
+        self.param1 = param1
+        self.param2 = param2
+    
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table):
+        # Use the built-in merging function
+        combined_result = self.merge_results(vector_results, fts_results)
+        
+        # Do something with the combined results
+        # ...
+
+        # Return the combined results
+        return combined_result
+
+```
+
+### Example of a Custom Reranker
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
+
+```python
+
+from typing import List, Union
+import pandas as pd
+from lancedb.rerankers import CohereReranker
+
+class MofidifiedCohereReranker(CohereReranker):
+    def __init__(self, filters: Union[str, List[str]], **kwargs):
+        super().__init__(**kwargs)
+        filters = filters if isinstance(filters, list) else [filters]
+        self.filters = filters
+
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
+        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
+        df = combined_result.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
+
+```
+
+!!! tip
+    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/src/integrations/index.md

@@ -13,7 +13,7 @@ Get started using these examples and quick links.
 | Integrations | |
 |---|---:|
 | <h3> LlamaIndex </h3>LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. <h3>[Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html) </h3> |<img src="../assets/llama-index.jpg" alt="image" width="150" height="auto">|
-| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://python.langchain.com/docs/integrations/vectorstores/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
+| <h3>Langchain</h3>Langchain allows building applications with LLMs through composability <h3>[Lean More](https://lancedb.github.io/lancedb/integrations/langchain/) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Langchain TS</h3> Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. <h3>[Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | <img src="../assets/langchain.png" alt="image" width="150" height="auto">|
 | <h3>Voxel51</h3>  It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.<h3>[Learn More](./voxel51.md) | <img src="../assets/voxel.gif" alt="image" width="150" height="auto">|
 | <h3>PromptTools</h3>  Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.<h3>[Learn More](./prompttools.md) | <img src="../assets/prompttools.jpeg" alt="image" width="150" height="auto">|

docs/src/integrations/langchain.md

@@ -0,0 +1,201 @@
+# Langchain
+![Illustration](../assets/langchain.png)
+
+## Quick Start
+You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
+```python
+import os
+from langchain.document_loaders import TextLoader
+from langchain.vectorstores import LanceDB
+from langchain_openai import OpenAIEmbeddings
+from langchain_text_splitters import CharacterTextSplitter
+
+os.environ["OPENAI_API_KEY"] = "sk-..."
+
+loader = TextLoader("../../modules/state_of_the_union.txt") # Replace with your data path
+documents = loader.load()
+
+documents = CharacterTextSplitter().split_documents(documents)
+embeddings = OpenAIEmbeddings()
+
+docsearch = LanceDB.from_documents(documents, embeddings)
+query = "What did the president say about Ketanji Brown Jackson"
+docs = docsearch.similarity_search(query)
+print(docs[0].page_content)
+```
+
+## Documentation
+In the above example `LanceDB` vector store class object is created using `from_documents()` method  which is a `classmethod` and returns the initialized class object. 
+You can also use `LanceDB.from_texts(texts: List[str],embedding: Embeddings)` class method.  
+
+The exhaustive list of parameters for `LanceDB` vector store are :  
+- `connection`: (Optional) `lancedb.db.LanceDBConnection` connection object to use.  If not provided, a new connection will be created.  
+- `embedding`: Langchain embedding model.  
+- `vector_key`: (Optional) Column name to use for vector's in the table. Defaults to `'vector'`.   
+- `id_key`: (Optional) Column name to use for id's in the table. Defaults to `'id'`.  
+- `text_key`: (Optional) Column name to use for text in the table. Defaults to `'text'`.  
+- `table_name`: (Optional) Name of your table in the database. Defaults to `'vectorstore'`.  
+- `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
+- `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+- `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
+- `reranker`: (Optional) The reranker to use for LanceDB.
+- `relevance_score_fn`: (Optional[Callable[[float], float]]) Langchain relevance score function to be used. Defaults to `None`. 
+
+```python
+db_url = "db://lang_test" # url of db you created
+api_key = "xxxxx" # your API key
+region="us-east-1-dev"  # your selected region
+
+vector_store = LanceDB(
+    uri=db_url,
+    api_key=api_key, #(dont include for local API)
+    region=region, #(dont include for local API)
+    embedding=embeddings,
+    table_name='langchain_test' #Optional
+    )
+```
+
+### Methods 
+
+##### add_texts()
+- `texts`: `Iterable` of strings to add to the vectorstore.
+- `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
+- `ids`: Optional `list` of ids to associate with the texts. 
+- `kwargs`: `Any`
+
+This method adds texts and stores respective embeddings automatically.
+
+```python
+vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
+
+#Additionaly, to explore the table you can load it into a df or save it in a csv file:
+
+tbl = vector_store.get_table()
+print("tbl:", tbl)
+pd_df = tbl.to_pandas()
+pd_df.to_csv("docsearch.csv", index=False)
+
+# you can also create a new vector store object using an older connection object:
+vector_store = LanceDB(connection=tbl, embedding=embeddings)
+```
+##### create_index() 
+- `col_name`: `Optional[str] = None`
+- `vector_col`: `Optional[str] = None`
+- `num_partitions`: `Optional[int] = 256`
+- `num_sub_vectors`: `Optional[int] = 96`
+- `index_cache_size`: `Optional[int] = None`
+
+This method creates an index for the vector store. For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+
+```python
+# for creating vector index
+vector_store.create_index(vector_col='vector', metric = 'cosine')
+
+# for creating scalar index(for non-vector columns)
+vector_store.create_index(col_name='text')
+
+```
+
+##### similarity_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `fts`: `Optional[bool] = False`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query without relevance scores
+
+```python
+docs = docsearch.similarity_search(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_by_vector()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query vector.
+
+```python
+docs = docsearch.similarity_search_by_vector(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_with_score()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query string with relevance scores, gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+
+```python
+docs = docsearch.similarity_search_with_relevance_scores(query)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### similarity_search_by_vector_with_relevance_scores()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query vector with relevance scores.
+Relevance score 
+
+```python
+docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### max_marginal_relevance_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `fetch_k` : Number of Documents to fetch to pass to MMR algorithm, `Optional[int] = None`
+- `lambda_mult`: Number between 0 and 1 that determines the degree
+                        of diversity among the results with 0 corresponding
+                        to maximum diversity and 1 to minimum diversity.
+                        Defaults to 0.5. `float = 0.5`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns docs selected using the maximal marginal relevance(MMR).
+Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
+
+Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
+
+```python
+result = docsearch.max_marginal_relevance_search(
+        query="text"
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+
+## search by vector :
+result = docsearch.max_marginal_relevance_search_by_vector(
+        embeddings.embed_query("text")
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+```
+
+##### add_images()
+- `uris` : File path to the image. `List[str]`.
+- `metadatas` : Optional list of metadatas. `(Optional[List[dict]], optional)`
+- `ids` : Optional list of IDs. `(Optional[List[str]], optional)`
+
+Adds images by automatically creating their embeddings and adds them to the vectorstore.
+
+```python
+vec_store.add_images(uris=image_uris) 
+# here image_uris are local fs paths to the images.
+```
+
+

docs/src/integrations/llamaIndex.md

@@ -0,0 +1,142 @@
+# Llama-Index
+![Illustration](../assets/llama-index.jpg)
+
+## Quick start
+You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. 
+You can run the below script to try it out :
+```python
+import logging
+import sys
+
+# Uncomment to see debug logs
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
+
+from llama_index.core import SimpleDirectoryReader, Document, StorageContext
+from llama_index.core import VectorStoreIndex
+from llama_index.vector_stores.lancedb import LanceDBVectorStore
+import textwrap
+import openai
+
+openai.api_key = "sk-..."
+
+documents = SimpleDirectoryReader("./data/your-data-dir/").load_data()
+print("Document ID:", documents[0].doc_id, "Document Hash:", documents[0].hash)
+
+## For LanceDB cloud :
+# vector_store = LanceDBVectorStore( 
+#     uri="db://db_name", # your remote DB URI
+#     api_key="sk_..", # lancedb cloud api key
+#     region="your-region" # the region you configured
+#     ...
+# )
+
+vector_store = LanceDBVectorStore(
+    uri="./lancedb", mode="overwrite", query_type="vector"
+)
+storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+index = VectorStoreIndex.from_documents(
+    documents, storage_context=storage_context
+)
+lance_filter = "metadata.file_name = 'paul_graham_essay.txt' "
+retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
+response = retriever.retrieve("What did the author do growing up?")
+```
+
+Checkout Complete example here - [LlamaIndex demo](../notebooks/LlamaIndex_example.ipynb)
+
+### Filtering
+For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
+```python
+from llama_index.core.vector_stores import (
+    MetadataFilters,
+    FilterOperator,
+    FilterCondition,
+    MetadataFilter,
+)
+
+query_filters = MetadataFilters(
+    filters=[
+        MetadataFilter(
+            key="creation_date", operator=FilterOperator.EQ, value="2024-05-23"
+        ),
+        MetadataFilter(
+            key="file_size", value=75040, operator=FilterOperator.GT
+        ),
+    ],
+    condition=FilterCondition.AND,
+)
+```
+
+### Hybrid Search
+For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose.
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+vector_store._add_reranker(reranker)
+
+query_engine = index.as_query_engine(
+    filters=query_filters,
+    vector_store_kwargs={
+        "query_type": "hybrid",
+    }
+)
+
+response = query_engine.query("How much did Viaweb charge per month?")
+```
+
+In the above snippet, you can change/specify query_type again when creating the engine/retriever.
+
+## API reference
+The exhaustive list of parameters for `LanceDBVectorStore` vector store are :  
+- `connection`: Optional, `lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.
+- `uri`: Optional[str], the uri of your database. Defaults to `"/tmp/lancedb"`.
+- `table_name` : Optional[str], Name of your table in the database. Defaults to `"vectors"`.
+- `table`: Optional[Any], `lancedb.db.LanceTable` object to be passed. Defaults to `None`. 
+- `vector_column_name`: Optional[Any], Column name to use for vector's in the table. Defaults to `'vector'`.   
+- `doc_id_key`: Optional[str], Column name to use for document id's in the table. Defaults to `'doc_id'`.  
+- `text_key`: Optional[str], Column name to use for text in the table. Defaults to `'text'`.  
+- `api_key`: Optional[str], API key to use for LanceDB cloud database. Defaults to `None`.  
+- `region`: Optional[str], Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+- `nprobes` : Optional[int], Set the number of probes to use. Only applicable if ANN index is created on the table else its ignored. Defaults to `20`.
+- `refine_factor` : Optional[int], Refine the results by reading extra elements and re-ranking them in memory. Defaults to `None`.
+- `reranker`: Optional[Any], The reranker to use for LanceDB.
+        Defaults to `None`.
+- `overfetch_factor`: Optional[int], The factor by which to fetch more results.
+        Defaults to `1`.
+- `mode`: Optional[str], The mode to use for LanceDB.
+            Defaults to `"overwrite"`.
+- `query_type`:Optional[str], The type of query to use for LanceDB.
+            Defaults to `"vector"`.
+
+
+### Methods 
+
+- __from_table(cls, table: lancedb.db.LanceTable) -> `LanceDBVectorStore`__ : (class method) Creates instance from lancedb table. 
+
+- **_add_reranker(self, reranker: lancedb.rerankers.Reranker) -> `None`** : Add a reranker to an existing vector store. 
+    - Usage :
+        ```python
+        from lancedb.rerankers import ColbertReranker
+        reranker = ColbertReranker()
+        vector_store._add_reranker(reranker)
+        ```               
+- **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
+- __create_index(  
+  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
+) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
+        Make sure your vector column has enough data before creating an index on it.
+
+- __add(self, nodes: `List[BaseNode]`, **add_kwargs: `Any`, ) -> `List[str]`__ :
+adds Nodes to the table
+
+- **delete(self, ref_doc_id: `str`) -> `None`**: Delete nodes using with node_ids.
+- **delete_nodes(self, node_ids: `List[str]`) -> `None`** : Delete nodes using with node_ids.
+- __query(
+        self,
+        query: `VectorStoreQuery`,
+        **kwargs: `Any`,
+    ) -> `VectorStoreQueryResult`__:
+        Query index(`VectorStoreIndex`) for top k most similar nodes. Accepts llamaIndex `VectorStoreQuery` object.

docs/src/js/.nojekyll

@@ -0,0 +1 @@
+TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false.

docs/src/js/README.md

@@ -0,0 +1,76 @@
+**@lancedb/lancedb** • [**Docs**](globals.md)
+
+***
+
+# LanceDB JavaScript SDK
+
+A JavaScript library for [LanceDB](https://github.com/lancedb/lancedb).
+
+## Installation
+
+```bash
+npm install @lancedb/lancedb
+```
+
+This will download the appropriate native library for your platform. We currently
+support:
+
+- Linux (x86_64 and aarch64)
+- MacOS (Intel and ARM/M1/M2)
+- Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
+
+## Usage
+
+### Basic Example
+
+```javascript
+import * as lancedb from "@lancedb/lancedb";
+const db = await lancedb.connect("data/sample-lancedb");
+const table = await db.createTable("my_table", [
+  { id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+  { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 },
+]);
+const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
+console.log(results);
+```
+
+The [quickstart](../basic.md) contains a more complete example.
+
+## Development
+
+```sh
+npm run build
+npm run test
+```
+
+### Running lint / format
+
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
+To manually lint your code you can run:
+
+```sh
+npm run lint
+```
+
+to automatically fix all fixable issues:
+
+```sh
+npm run lint-fix
+```
+
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
+
+### Generating docs
+
+```sh
+npm run docs
+
+cd ../docs
+# Asssume the virtual environment was created
+# python3 -m venv venv
+# pip install -r requirements.txt
+. ./venv/bin/activate
+mkdocs build
+```

docs/src/js/classes/Connection.md

@@ -0,0 +1,200 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Connection
+
+# Class: `abstract` Connection
+
+A LanceDB Connection that allows you to open tables and create new ones.
+
+Connection could be local against filesystem or remote against a server.
+
+A Connection is intended to be a long lived object and may hold open
+resources such as HTTP connection pools.  This is generally fine and
+a single connection should be shared if it is going to be used many
+times. However, if you are finished with a connection, you may call
+close to eagerly free these resources.  Any call to a Connection
+method after it has been closed will result in an error.
+
+Closing a connection is optional.  Connections will automatically
+be closed when they are garbage collected.
+
+Any created tables are independent and will continue to work even if
+the underlying connection has been closed.
+
+## Constructors
+
+### new Connection()
+
+> **new Connection**(): [`Connection`](Connection.md)
+
+#### Returns
+
+[`Connection`](Connection.md)
+
+## Methods
+
+### close()
+
+> `abstract` **close**(): `void`
+
+Close the connection, releasing any underlying resources.
+
+It is safe to call this method multiple times.
+
+Any attempt to use the connection after it is closed will result in an error.
+
+#### Returns
+
+`void`
+
+***
+
+### createEmptyTable()
+
+> `abstract` **createEmptyTable**(`name`, `schema`, `options`?): `Promise`<[`Table`](Table.md)>
+
+Creates a new empty Table
+
+#### Parameters
+
+• **name**: `string`
+
+The name of the table.
+
+• **schema**: `SchemaLike`
+
+The schema of the table
+
+• **options?**: `Partial`<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)>
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)>
+
+***
+
+### createTable()
+
+#### createTable(options)
+
+> `abstract` **createTable**(`options`): `Promise`<[`Table`](Table.md)>
+
+Creates a new Table and initialize it with new data.
+
+##### Parameters
+
+• **options**: `object` & `Partial`<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)>
+
+The options object.
+
+##### Returns
+
+`Promise`<[`Table`](Table.md)>
+
+#### createTable(name, data, options)
+
+> `abstract` **createTable**(`name`, `data`, `options`?): `Promise`<[`Table`](Table.md)>
+
+Creates a new Table and initialize it with new data.
+
+##### Parameters
+
+• **name**: `string`
+
+The name of the table.
+
+• **data**: `TableLike` \| `Record`<`string`, `unknown`>[]
+
+Non-empty Array of Records
+to be inserted into the table
+
+• **options?**: `Partial`<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)>
+
+##### Returns
+
+`Promise`<[`Table`](Table.md)>
+
+***
+
+### display()
+
+> `abstract` **display**(): `string`
+
+Return a brief description of the connection
+
+#### Returns
+
+`string`
+
+***
+
+### dropTable()
+
+> `abstract` **dropTable**(`name`): `Promise`<`void`>
+
+Drop an existing table.
+
+#### Parameters
+
+• **name**: `string`
+
+The name of the table to drop.
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### isOpen()
+
+> `abstract` **isOpen**(): `boolean`
+
+Return true if the connection has not been closed
+
+#### Returns
+
+`boolean`
+
+***
+
+### openTable()
+
+> `abstract` **openTable**(`name`, `options`?): `Promise`<[`Table`](Table.md)>
+
+Open a table in the database.
+
+#### Parameters
+
+• **name**: `string`
+
+The name of the table
+
+• **options?**: `Partial`<`OpenTableOptions`>
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)>
+
+***
+
+### tableNames()
+
+> `abstract` **tableNames**(`options`?): `Promise`<`string`[]>
+
+List all the table names in this database.
+
+Tables will be returned in lexicographical order.
+
+#### Parameters
+
+• **options?**: `Partial`<[`TableNamesOptions`](../interfaces/TableNamesOptions.md)>
+
+options to control the
+paging / start point
+
+#### Returns
+
+`Promise`<`string`[]>

docs/src/js/classes/Index.md

@@ -0,0 +1,70 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Index
+
+# Class: Index
+
+## Methods
+
+### btree()
+
+> `static` **btree**(): [`Index`](Index.md)
+
+Create a btree index
+
+A btree index is an index on a scalar columns.  The index stores a copy of the column
+in sorted order.  A header entry is created for each block of rows (currently the
+block size is fixed at 4096).  These header entries are stored in a separate
+cacheable structure (a btree).  To search for data the header is used to determine
+which blocks need to be read from disk.
+
+For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+the correct row ids.
+
+This index is good for scalar columns with mostly distinct values and does best when
+the query is highly selective.
+
+The btree index does not currently have any parameters though parameters such as the
+block size may be added in the future.
+
+#### Returns
+
+[`Index`](Index.md)
+
+***
+
+### ivfPq()
+
+> `static` **ivfPq**(`options`?): [`Index`](Index.md)
+
+Create an IvfPq index
+
+This index stores a compressed (quantized) copy of every vector.  These vectors
+are grouped into partitions of similar vectors.  Each partition keeps track of
+a centroid which is the average value of all vectors in the group.
+
+During a query the centroids are compared with the query vector to find the closest
+partitions.  The compressed vectors in these partitions are then searched to find
+the closest vectors.
+
+The compression scheme is called product quantization.  Each vector is divided into
+subvectors and then each subvector is quantized into a small number of bits.  the
+parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+between index size (and thus search speed) and index accuracy.
+
+The partitioning process is called IVF and the `num_partitions` parameter controls how
+many groups to create.
+
+Note that training an IVF PQ index on a large dataset is a slow operation and
+currently is also a memory intensive operation.
+
+#### Parameters
+
+• **options?**: `Partial`<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)>
+
+#### Returns
+
+[`Index`](Index.md)

docs/src/js/classes/MakeArrowTableOptions.md

@@ -0,0 +1,61 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MakeArrowTableOptions
+
+# Class: MakeArrowTableOptions
+
+Options to control the makeArrowTable call.
+
+## Constructors
+
+### new MakeArrowTableOptions()
+
+> **new MakeArrowTableOptions**(`values`?): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
+
+#### Parameters
+
+• **values?**: `Partial`<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)>
+
+#### Returns
+
+[`MakeArrowTableOptions`](MakeArrowTableOptions.md)
+
+## Properties
+
+### dictionaryEncodeStrings
+
+> **dictionaryEncodeStrings**: `boolean` = `false`
+
+If true then string columns will be encoded with dictionary encoding
+
+Set this to true if your string columns tend to repeat the same values
+often.  For more precise control use the `schema` property to specify the
+data type for individual columns.
+
+If `schema` is provided then this property is ignored.
+
+***
+
+### embeddingFunction?
+
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
+
+***
+
+### embeddings?
+
+> `optional` **embeddings**: [`EmbeddingFunction`](../namespaces/embedding/classes/EmbeddingFunction.md)<`unknown`, `FunctionOptions`>
+
+***
+
+### schema?
+
+> `optional` **schema**: `SchemaLike`
+
+***
+
+### vectorColumns
+
+> **vectorColumns**: `Record`<`string`, [`VectorColumnOptions`](VectorColumnOptions.md)>

docs/src/js/classes/Query.md

@@ -0,0 +1,391 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Query
+
+# Class: Query
+
+A builder for LanceDB queries.
+
+## Extends
+
+- [`QueryBase`](QueryBase.md)<`NativeQuery`>
+
+## Constructors
+
+### new Query()
+
+> **new Query**(`tbl`): [`Query`](Query.md)
+
+#### Parameters
+
+• **tbl**: `Table`
+
+#### Returns
+
+[`Query`](Query.md)
+
+#### Overrides
+
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
+
+## Properties
+
+### inner
+
+> `protected` **inner**: `Query` \| `Promise`<`Query`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+
+## Methods
+
+### \[asyncIterator\]()
+
+> **\[asyncIterator\]**(): `AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Returns
+
+`AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)
+
+***
+
+### doCall()
+
+> `protected` **doCall**(`fn`): `void`
+
+#### Parameters
+
+• **fn**
+
+#### Returns
+
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+
+Execute the query and return the results as an
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+[`RecordBatchIterator`](RecordBatchIterator.md)
+
+#### See
+
+ - AsyncIterator
+of
+ - RecordBatch.
+
+By default, LanceDb will use many threads to calculate results and, when
+the result set is large, multiple batches will be processed at one time.
+This readahead is limited however and backpressure will be applied if this
+stream is consumed slowly (this constrains the maximum memory used by a
+single query)
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
+
+***
+
+### explainPlan()
+
+> **explainPlan**(`verbose`): `Promise`<`string`>
+
+Generates an explanation of the query execution plan.
+
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`<`string`>
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
+
+Set the maximum number of results to return.
+
+By default, a plain search has no limit.  If this method is not
+called then every valid row from the table will be returned.
+
+#### Parameters
+
+• **limit**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
+
+***
+
+### nativeExecute()
+
+> `protected` **nativeExecute**(`options`?): `Promise`<`RecordBatchIterator`>
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+`Promise`<`RecordBatchIterator`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
+
+***
+
+### nearestTo()
+
+> **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)
+
+Find the nearest vectors to the given query vector.
+
+This converts the query from a plain query to a vector query.
+
+This method will attempt to convert the input to the query vector
+expected by the embedding model.  If the input cannot be converted
+then an error will be thrown.
+
+By default, there is no embedding model, and the input should be
+an array-like object of numbers (something that can be used as input
+to Float32Array.from)
+
+If there is only one vector column (a column whose data type is a
+fixed size list of floats) then the column does not need to be specified.
+If there is more than one vector column you must use
+
+#### Parameters
+
+• **vector**: `IntoVector`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### See
+
+ - [VectorQuery#column](VectorQuery.md#column)  to specify which column you would like
+to compare with.
+
+If no index has been created on the vector column then a vector query
+will perform a distance comparison between the query vector and every
+vector in the database and then sort the results.  This is sometimes
+called a "flat search"
+
+For small databases, with a few hundred thousand vectors or less, this can
+be reasonably fast.  In larger databases you should create a vector index
+on the column.  If there is a vector index then an "approximate" nearest
+neighbor search (frequently called an ANN search) will be performed.  This
+search is much faster, but the results will be approximate.
+
+The query can be further parameterized using the returned builder.  There
+are various ANN search parameters that will let you fine tune your recall
+accuracy vs search latency.
+
+Vector searches always have a `limit`.  If `limit` has not been called then
+a default `limit` of 10 will be used.
+ - [Query#limit](Query.md#limit)
+
+***
+
+### select()
+
+> **select**(`columns`): `this`
+
+Return only the specified columns.
+
+By default a query will return all columns from the table.  However, this can have
+a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+means we can finely tune our I/O to select exactly the columns we need.
+
+As a best practice you should always limit queries to the columns that you need.  If you
+pass in an array of column names then only those columns will be returned.
+
+You can also use this method to create new "dynamic" columns based on your existing columns.
+For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+
+To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+for each entry in the map.  The key provides the name of the column.  The value is
+an SQL string used to specify how the column is calculated.
+
+For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+input to this method would be:
+
+#### Parameters
+
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+new Map([["combined", "a + b"], ["c", "c"]])
+
+Columns will always be returned in the order given, even if that order is different than
+the order used when adding the data.
+
+Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+uses `Object.entries` which should preserve the insertion order of the object.  However,
+object insertion order is easy to get wrong and `Map` is more foolproof.
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
+
+***
+
+### toArray()
+
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+
+Collect the results as an array of objects.
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`any`[]&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
+
+***
+
+### toArrow()
+
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+Collect the results as an Arrow
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+#### See
+
+ArrowTable.
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
+
+***
+
+### where()
+
+> **where**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+The filter should be supplied as an SQL query string.  For example:
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+x > 10
+y > 0 AND y < 100
+x > 5 OR y = 'test'
+
+Filtering performance can often be improved by creating a scalar index
+on the filter column(s).
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)

docs/src/js/classes/QueryBase.md

@@ -0,0 +1,298 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / QueryBase
+
+# Class: QueryBase<NativeQueryType>
+
+Common methods supported by all query types
+
+## Extended by
+
+- [`Query`](Query.md)
+- [`VectorQuery`](VectorQuery.md)
+
+## Type Parameters
+
+• **NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery`
+
+## Implements
+
+- `AsyncIterable`<`RecordBatch`>
+
+## Constructors
+
+### new QueryBase()
+
+> `protected` **new QueryBase**<`NativeQueryType`>(`inner`): [`QueryBase`](QueryBase.md)<`NativeQueryType`>
+
+#### Parameters
+
+• **inner**: `NativeQueryType` \| `Promise`<`NativeQueryType`>
+
+#### Returns
+
+[`QueryBase`](QueryBase.md)<`NativeQueryType`>
+
+## Properties
+
+### inner
+
+> `protected` **inner**: `NativeQueryType` \| `Promise`<`NativeQueryType`>
+
+## Methods
+
+### \[asyncIterator\]()
+
+> **\[asyncIterator\]**(): `AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Returns
+
+`AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Implementation of
+
+`AsyncIterable.[asyncIterator]`
+
+***
+
+### doCall()
+
+> `protected` **doCall**(`fn`): `void`
+
+#### Parameters
+
+• **fn**
+
+#### Returns
+
+`void`
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+
+Execute the query and return the results as an
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+[`RecordBatchIterator`](RecordBatchIterator.md)
+
+#### See
+
+ - AsyncIterator
+of
+ - RecordBatch.
+
+By default, LanceDb will use many threads to calculate results and, when
+the result set is large, multiple batches will be processed at one time.
+This readahead is limited however and backpressure will be applied if this
+stream is consumed slowly (this constrains the maximum memory used by a
+single query)
+
+***
+
+### explainPlan()
+
+> **explainPlan**(`verbose`): `Promise`<`string`>
+
+Generates an explanation of the query execution plan.
+
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`<`string`>
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
+
+Set the maximum number of results to return.
+
+By default, a plain search has no limit.  If this method is not
+called then every valid row from the table will be returned.
+
+#### Parameters
+
+• **limit**: `number`
+
+#### Returns
+
+`this`
+
+***
+
+### nativeExecute()
+
+> `protected` **nativeExecute**(`options`?): `Promise`<`RecordBatchIterator`>
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+`Promise`<`RecordBatchIterator`>
+
+***
+
+### select()
+
+> **select**(`columns`): `this`
+
+Return only the specified columns.
+
+By default a query will return all columns from the table.  However, this can have
+a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+means we can finely tune our I/O to select exactly the columns we need.
+
+As a best practice you should always limit queries to the columns that you need.  If you
+pass in an array of column names then only those columns will be returned.
+
+You can also use this method to create new "dynamic" columns based on your existing columns.
+For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+
+To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+for each entry in the map.  The key provides the name of the column.  The value is
+an SQL string used to specify how the column is calculated.
+
+For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+input to this method would be:
+
+#### Parameters
+
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+new Map([["combined", "a + b"], ["c", "c"]])
+
+Columns will always be returned in the order given, even if that order is different than
+the order used when adding the data.
+
+Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+uses `Object.entries` which should preserve the insertion order of the object.  However,
+object insertion order is easy to get wrong and `Map` is more foolproof.
+```
+
+***
+
+### toArray()
+
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+
+Collect the results as an array of objects.
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`any`[]&gt;
+
+***
+
+### toArrow()
+
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+Collect the results as an Arrow
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+#### See
+
+ArrowTable.
+
+***
+
+### where()
+
+> **where**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+The filter should be supplied as an SQL query string.  For example:
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+x > 10
+y > 0 AND y < 100
+x > 5 OR y = 'test'
+
+Filtering performance can often be improved by creating a scalar index
+on the filter column(s).
+```

docs/src/js/classes/RecordBatchIterator.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RecordBatchIterator
+
+# Class: RecordBatchIterator
+
+## Implements
+
+- `AsyncIterator`<`RecordBatch`>
+
+## Constructors
+
+### new RecordBatchIterator()
+
+> **new RecordBatchIterator**(`promise`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+
+#### Parameters
+
+• **promise?**: `Promise`<`RecordBatchIterator`>
+
+#### Returns
+
+[`RecordBatchIterator`](RecordBatchIterator.md)
+
+## Methods
+
+### next()
+
+> **next**(): `Promise`<`IteratorResult`<`RecordBatch`<`any`>, `any`>>
+
+#### Returns
+
+`Promise`<`IteratorResult`<`RecordBatch`<`any`>, `any`>>
+
+#### Implementation of
+
+`AsyncIterator.next`

docs/src/js/classes/Table.md

@@ -0,0 +1,696 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Table
+
+# Class: `abstract` Table
+
+A Table is a collection of Records in a LanceDB Database.
+
+A Table object is expected to be long lived and reused for multiple operations.
+Table objects will cache a certain amount of index data in memory.  This cache
+will be freed when the Table is garbage collected.  To eagerly free the cache you
+can call the `close` method.  Once the Table is closed, it cannot be used for any
+further operations.
+
+Closing a table is optional.  It not closed, it will be closed when it is garbage
+collected.
+
+## Constructors
+
+### new Table()
+
+> **new Table**(): [`Table`](Table.md)
+
+#### Returns
+
+[`Table`](Table.md)
+
+## Accessors
+
+### name
+
+> `get` `abstract` **name**(): `string`
+
+Returns the name of the table
+
+#### Returns
+
+`string`
+
+## Methods
+
+### add()
+
+> `abstract` **add**(`data`, `options`?): `Promise`<`void`>
+
+Insert records into this Table.
+
+#### Parameters
+
+• **data**: [`Data`](../type-aliases/Data.md)
+
+Records to be inserted into the Table
+
+• **options?**: `Partial`<[`AddDataOptions`](../interfaces/AddDataOptions.md)>
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### addColumns()
+
+> `abstract` **addColumns**(`newColumnTransforms`): `Promise`<`void`>
+
+Add new columns with defined values.
+
+#### Parameters
+
+• **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
+
+pairs of column names and
+the SQL expression to use to calculate the value of the new column. These
+expressions will be evaluated for each row in the table, and can
+reference existing columns in the table.
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### alterColumns()
+
+> `abstract` **alterColumns**(`columnAlterations`): `Promise`<`void`>
+
+Alter the name or nullability of columns.
+
+#### Parameters
+
+• **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
+
+One or more alterations to
+apply to columns.
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### checkout()
+
+> `abstract` **checkout**(`version`): `Promise`<`void`>
+
+Checks out a specific version of the table _This is an in-place operation._
+
+This allows viewing previous versions of the table. If you wish to
+keep writing to the dataset starting from an old version, then use
+the `restore` function.
+
+Calling this method will set the table into time-travel mode. If you
+wish to return to standard mode, call `checkoutLatest`.
+
+#### Parameters
+
+• **version**: `number`
+
+The version to checkout
+
+#### Returns
+
+`Promise`<`void`>
+
+#### Example
+
+```typescript
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], type: "vector" },
+]);
+
+console.log(await table.version()); // 1
+console.log(table.display());
+await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
+await table.checkout(1);
+console.log(await table.version()); // 2
+```
+
+***
+
+### checkoutLatest()
+
+> `abstract` **checkoutLatest**(): `Promise`<`void`>
+
+Checkout the latest version of the table. _This is an in-place operation._
+
+The table will be set back into standard mode, and will track the latest
+version of the table.
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### close()
+
+> `abstract` **close**(): `void`
+
+Close the table, releasing any underlying resources.
+
+It is safe to call this method multiple times.
+
+Any attempt to use the table after it is closed will result in an error.
+
+#### Returns
+
+`void`
+
+***
+
+### countRows()
+
+> `abstract` **countRows**(`filter`?): `Promise`<`number`>
+
+Count the total number of rows in the dataset.
+
+#### Parameters
+
+• **filter?**: `string`
+
+#### Returns
+
+`Promise`<`number`>
+
+***
+
+### createIndex()
+
+> `abstract` **createIndex**(`column`, `options`?): `Promise`<`void`>
+
+Create an index to speed up queries.
+
+Indices can be created on vector columns or scalar columns.
+Indices on vector columns will speed up vector searches.
+Indices on scalar columns will speed up filtering (in both
+vector and non-vector searches)
+
+#### Parameters
+
+• **column**: `string`
+
+• **options?**: `Partial`<[`IndexOptions`](../interfaces/IndexOptions.md)>
+
+#### Returns
+
+`Promise`<`void`>
+
+#### Note
+
+We currently don't support custom named indexes,
+The index name will always be `${column}_idx`
+
+#### Examples
+
+```ts
+// If the column has a vector (fixed size list) data type then
+// an IvfPq vector index will be created.
+const table = await conn.openTable("my_table");
+await table.createIndex("vector");
+```
+
+```ts
+// For advanced control over vector index creation you can specify
+// the index type and options.
+const table = await conn.openTable("my_table");
+await table.createIndex("vector", {
+  config: lancedb.Index.ivfPq({
+    numPartitions: 128,
+    numSubVectors: 16,
+  }),
+});
+```
+
+```ts
+// Or create a Scalar index
+await table.createIndex("my_float_col");
+```
+
+***
+
+### delete()
+
+> `abstract` **delete**(`predicate`): `Promise`<`void`>
+
+Delete the rows that satisfy the predicate.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### display()
+
+> `abstract` **display**(): `string`
+
+Return a brief description of the table
+
+#### Returns
+
+`string`
+
+***
+
+### dropColumns()
+
+> `abstract` **dropColumns**(`columnNames`): `Promise`<`void`>
+
+Drop one or more columns from the dataset
+
+This is a metadata-only operation and does not remove the data from the
+underlying storage. In order to remove the data, you must subsequently
+call ``compact_files`` to rewrite the data without the removed columns and
+then call ``cleanup_files`` to remove the old files.
+
+#### Parameters
+
+• **columnNames**: `string`[]
+
+The names of the columns to drop. These can
+be nested column references (e.g. "a.b.c") or top-level column names
+(e.g. "a").
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### indexStats()
+
+> `abstract` **indexStats**(`name`): `Promise`<`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)>
+
+List all the stats of a specified index
+
+#### Parameters
+
+• **name**: `string`
+
+The name of the index.
+
+#### Returns
+
+`Promise`<`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)>
+
+The stats of the index. If the index does not exist, it will return undefined
+
+***
+
+### isOpen()
+
+> `abstract` **isOpen**(): `boolean`
+
+Return true if the table has not been closed
+
+#### Returns
+
+`boolean`
+
+***
+
+### listIndices()
+
+> `abstract` **listIndices**(): `Promise`<[`IndexConfig`](../interfaces/IndexConfig.md)[]>
+
+List all indices that have been created with [Table.createIndex](Table.md#createindex)
+
+#### Returns
+
+`Promise`<[`IndexConfig`](../interfaces/IndexConfig.md)[]>
+
+***
+
+### mergeInsert()
+
+> `abstract` **mergeInsert**(`on`): `MergeInsertBuilder`
+
+#### Parameters
+
+• **on**: `string` \| `string`[]
+
+#### Returns
+
+`MergeInsertBuilder`
+
+***
+
+### optimize()
+
+> `abstract` **optimize**(`options`?): `Promise`<`OptimizeStats`>
+
+Optimize the on-disk data and indices for better performance.
+
+Modeled after ``VACUUM`` in PostgreSQL.
+
+ Optimization covers three operations:
+
+ - Compaction: Merges small files into larger ones
+ - Prune: Removes old versions of the dataset
+ - Index: Optimizes the indices, adding new data to existing indices
+
+ Experimental API
+ ----------------
+
+ The optimization process is undergoing active development and may change.
+ Our goal with these changes is to improve the performance of optimization and
+ reduce the complexity.
+
+ That being said, it is essential today to run optimize if you want the best
+ performance.  It should be stable and safe to use in production, but it our
+ hope that the API may be simplified (or not even need to be called) in the
+ future.
+
+ The frequency an application shoudl call optimize is based on the frequency of
+ data modifications.  If data is frequently added, deleted, or updated then
+ optimize should be run frequently.  A good rule of thumb is to run optimize if
+ you have added or modified 100,000 or more records or run more than 20 data
+ modification operations.
+
+#### Parameters
+
+• **options?**: `Partial`<`OptimizeOptions`>
+
+#### Returns
+
+`Promise`<`OptimizeStats`>
+
+***
+
+### query()
+
+> `abstract` **query**(): [`Query`](Query.md)
+
+Create a [Query](Query.md) Builder.
+
+Queries allow you to search your existing data.  By default the query will
+return all the data in the table in no particular order.  The builder
+returned by this method can be used to control the query using filtering,
+vector similarity, sorting, and more.
+
+Note: By default, all columns are returned.  For best performance, you should
+only fetch the columns you need.
+
+When appropriate, various indices and statistics based pruning will be used to
+accelerate the query.
+
+#### Returns
+
+[`Query`](Query.md)
+
+A builder that can be used to parameterize the query
+
+#### Examples
+
+```ts
+// SQL-style filtering
+//
+// This query will return up to 1000 rows whose value in the `id` column
+// is greater than 5. LanceDb supports a broad set of filtering functions.
+for await (const batch of table
+  .query()
+  .where("id > 1")
+  .select(["id"])
+  .limit(20)) {
+  console.log(batch);
+}
+```
+
+```ts
+// Vector Similarity Search
+//
+// This example will find the 10 rows whose value in the "vector" column are
+// closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
+// on the "vector" column then this will perform an ANN search.
+//
+// The `refineFactor` and `nprobes` methods are used to control the recall /
+// latency tradeoff of the search.
+for await (const batch of table
+  .query()
+  .where("id > 1")
+  .select(["id"])
+  .limit(20)) {
+  console.log(batch);
+}
+```
+
+```ts
+// Scan the full dataset
+//
+// This query will return everything in the table in no particular order.
+for await (const batch of table.query()) {
+  console.log(batch);
+}
+```
+
+***
+
+### restore()
+
+> `abstract` **restore**(): `Promise`<`void`>
+
+Restore the table to the currently checked out version
+
+This operation will fail if checkout has not been called previously
+
+This operation will overwrite the latest version of the table with a
+previous version.  Any changes made since the checked out version will
+no longer be visible.
+
+Once the operation concludes the table will no longer be in a checked
+out state and the read_consistency_interval, if any, will apply.
+
+#### Returns
+
+`Promise`<`void`>
+
+***
+
+### schema()
+
+> `abstract` **schema**(): `Promise`<`Schema`<`any`>>
+
+Get the schema of the table.
+
+#### Returns
+
+`Promise`<`Schema`<`any`>>
+
+***
+
+### search()
+
+#### search(query)
+
+> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
+
+Create a search query to find the nearest neighbors
+of the given query vector
+
+##### Parameters
+
+• **query**: `string`
+
+the query. This will be converted to a vector using the table's provided embedding function
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+##### Note
+
+If no embedding functions are defined in the table, this will error when collecting the results.
+
+#### search(query)
+
+> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
+
+Create a search query to find the nearest neighbors
+of the given query vector
+
+##### Parameters
+
+• **query**: `IntoVector`
+
+the query vector
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
+### toArrow()
+
+> `abstract` **toArrow**(): `Promise`<`Table`<`any`>>
+
+Return the table as an arrow table
+
+#### Returns
+
+`Promise`<`Table`<`any`>>
+
+***
+
+### update()
+
+#### update(opts)
+
+> `abstract` **update**(`opts`): `Promise`<`void`>
+
+Update existing records in the Table
+
+##### Parameters
+
+• **opts**: `object` & `Partial`<[`UpdateOptions`](../interfaces/UpdateOptions.md)>
+
+##### Returns
+
+`Promise`<`void`>
+
+##### Example
+
+```ts
+table.update({where:"x = 2", values:{"vector": [10, 10]}})
+```
+
+#### update(opts)
+
+> `abstract` **update**(`opts`): `Promise`<`void`>
+
+Update existing records in the Table
+
+##### Parameters
+
+• **opts**: `object` & `Partial`<[`UpdateOptions`](../interfaces/UpdateOptions.md)>
+
+##### Returns
+
+`Promise`<`void`>
+
+##### Example
+
+```ts
+table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
+```
+
+#### update(updates, options)
+
+> `abstract` **update**(`updates`, `options`?): `Promise`<`void`>
+
+Update existing records in the Table
+
+An update operation can be used to adjust existing values.  Use the
+returned builder to specify which columns to update.  The new value
+can be a literal value (e.g. replacing nulls with some default value)
+or an expression applied to the old value (e.g. incrementing a value)
+
+An optional condition can be specified (e.g. "only update if the old
+value is 0")
+
+Note: if your condition is something like "some_id_column == 7" and
+you are updating many rows (with different ids) then you will get
+better performance with a single [`merge_insert`] call instead of
+repeatedly calilng this method.
+
+##### Parameters
+
+• **updates**: `Record`<`string`, `string`> \| `Map`<`string`, `string`>
+
+the
+columns to update
+
+Keys in the map should specify the name of the column to update.
+Values in the map provide the new value of the column.  These can
+be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+based on the row being updated (e.g. "my_col + 1")
+
+• **options?**: `Partial`<[`UpdateOptions`](../interfaces/UpdateOptions.md)>
+
+additional options to control
+the update behavior
+
+##### Returns
+
+`Promise`<`void`>
+
+***
+
+### vectorSearch()
+
+> `abstract` **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)
+
+Search the table with a given query vector.
+
+This is a convenience method for preparing a vector query and
+is the same thing as calling `nearestTo` on the builder returned
+by `query`.
+
+#### Parameters
+
+• **vector**: `IntoVector`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### See
+
+[Query#nearestTo](Query.md#nearestto) for more details.
+
+***
+
+### version()
+
+> `abstract` **version**(): `Promise`<`number`>
+
+Retrieve the version of the table
+
+#### Returns
+
+`Promise`<`number`>
+
+***
+
+### parseTableData()
+
+> `static` **parseTableData**(`data`, `options`?, `streaming`?): `Promise`<`object`>
+
+#### Parameters
+
+• **data**: `TableLike` \| `Record`<`string`, `unknown`>[]
+
+• **options?**: `Partial`<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)>
+
+• **streaming?**: `boolean` = `false`
+
+#### Returns
+
+`Promise`<`object`>
+
+##### buf
+
+> **buf**: `Buffer`
+
+##### mode
+
+> **mode**: `string`

docs/src/js/classes/VectorColumnOptions.md

@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / VectorColumnOptions
+
+# Class: VectorColumnOptions
+
+## Constructors
+
+### new VectorColumnOptions()
+
+> **new VectorColumnOptions**(`values`?): [`VectorColumnOptions`](VectorColumnOptions.md)
+
+#### Parameters
+
+• **values?**: `Partial`<[`VectorColumnOptions`](VectorColumnOptions.md)>
+
+#### Returns
+
+[`VectorColumnOptions`](VectorColumnOptions.md)
+
+## Properties
+
+### type
+
+> **type**: `Float`<`Floats`>
+
+Vector column type.

docs/src/js/classes/VectorQuery.md

@@ -0,0 +1,523 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / VectorQuery
+
+# Class: VectorQuery
+
+A builder used to construct a vector search
+
+This builder can be reused to execute the query many times.
+
+## Extends
+
+- [`QueryBase`](QueryBase.md)<`NativeVectorQuery`>
+
+## Constructors
+
+### new VectorQuery()
+
+> **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
+
+#### Parameters
+
+• **inner**: `VectorQuery` \| `Promise`<`VectorQuery`>
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### Overrides
+
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
+
+## Properties
+
+### inner
+
+> `protected` **inner**: `VectorQuery` \| `Promise`<`VectorQuery`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
+
+## Methods
+
+### \[asyncIterator\]()
+
+> **\[asyncIterator\]**(): `AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Returns
+
+`AsyncIterator`<`RecordBatch`<`any`>, `any`, `undefined`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)
+
+***
+
+### bypassVectorIndex()
+
+> **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)
+
+If this is called then any vector index is skipped
+
+An exhaustive (flat) search will be performed.  The query vector will
+be compared to every vector in the table.  At high scales this can be
+expensive.  However, this is often still useful.  For example, skipping
+the vector index can give you ground truth results which you can use to
+calculate your recall to select an appropriate value for nprobes.
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
+### column()
+
+> **column**(`column`): [`VectorQuery`](VectorQuery.md)
+
+Set the vector column to query
+
+This controls which column is compared to the query vector supplied in
+the call to
+
+#### Parameters
+
+• **column**: `string`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### See
+
+[Query#nearestTo](Query.md#nearestto)
+
+This parameter must be specified if the table has more than one column
+whose data type is a fixed-size-list of floats.
+
+***
+
+### distanceType()
+
+> **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)
+
+Set the distance metric to use
+
+When performing a vector search we try and find the "nearest" vectors according
+to some kind of distance metric.  This parameter controls which distance metric to
+use.  See
+
+#### Parameters
+
+• **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### See
+
+[IvfPqOptions.distanceType](../interfaces/IvfPqOptions.md#distancetype) for more details on the different
+distance metrics available.
+
+Note: if there is a vector index then the distance type used MUST match the distance
+type used to train the vector index.  If this is not done then the results will be
+invalid.
+
+By default "l2" is used.
+
+***
+
+### doCall()
+
+> `protected` **doCall**(`fn`): `void`
+
+#### Parameters
+
+• **fn**
+
+#### Returns
+
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
+
+Execute the query and return the results as an
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+[`RecordBatchIterator`](RecordBatchIterator.md)
+
+#### See
+
+ - AsyncIterator
+of
+ - RecordBatch.
+
+By default, LanceDb will use many threads to calculate results and, when
+the result set is large, multiple batches will be processed at one time.
+This readahead is limited however and backpressure will be applied if this
+stream is consumed slowly (this constrains the maximum memory used by a
+single query)
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
+
+***
+
+### explainPlan()
+
+> **explainPlan**(`verbose`): `Promise`<`string`>
+
+Generates an explanation of the query execution plan.
+
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`<`string`>
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
+
+Set the maximum number of results to return.
+
+By default, a plain search has no limit.  If this method is not
+called then every valid row from the table will be returned.
+
+#### Parameters
+
+• **limit**: `number`
+
+#### Returns
+
+`this`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
+
+***
+
+### nativeExecute()
+
+> `protected` **nativeExecute**(`options`?): `Promise`<`RecordBatchIterator`>
+
+#### Parameters
+
+• **options?**: `Partial`<`QueryExecutionOptions`>
+
+#### Returns
+
+`Promise`<`RecordBatchIterator`>
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
+
+***
+
+### nprobes()
+
+> **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)
+
+Set the number of partitions to search (probe)
+
+This argument is only used when the vector column has an IVF PQ index.
+If there is no index then this value is ignored.
+
+The IVF stage of IVF PQ divides the input into partitions (clusters) of
+related values.
+
+The partition whose centroids are closest to the query vector will be
+exhaustiely searched to find matches.  This parameter controls how many
+partitions should be searched.
+
+Increasing this value will increase the recall of your query but will
+also increase the latency of your query.  The default value is 20.  This
+default is good for many cases but the best value to use will depend on
+your data and the recall that you need to achieve.
+
+For best results we recommend tuning this parameter with a benchmark against
+your actual data to find the smallest possible value that will still give
+you the desired recall.
+
+#### Parameters
+
+• **nprobes**: `number`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
+### postfilter()
+
+> **postfilter**(): [`VectorQuery`](VectorQuery.md)
+
+If this is called then filtering will happen after the vector search instead of
+before.
+
+By default filtering will be performed before the vector search.  This is how
+filtering is typically understood to work.  This prefilter step does add some
+additional latency.  Creating a scalar index on the filter column(s) can
+often improve this latency.  However, sometimes a filter is too complex or scalar
+indices cannot be applied to the column.  In these cases postfiltering can be
+used instead of prefiltering to improve latency.
+
+Post filtering applies the filter to the results of the vector search.  This means
+we only run the filter on a much smaller set of data.  However, it can cause the
+query to return fewer than `limit` results (or even no results) if none of the nearest
+results match the filter.
+
+Post filtering happens during the "refine stage" (described in more detail in
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+#### See
+
+[VectorQuery#refineFactor](VectorQuery.md#refinefactor)).  This means that setting a higher refine
+factor can often help restore some of the results lost by post filtering.
+
+***
+
+### refineFactor()
+
+> **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)
+
+A multiplier to control how many additional rows are taken during the refine step
+
+This argument is only used when the vector column has an IVF PQ index.
+If there is no index then this value is ignored.
+
+An IVF PQ index stores compressed (quantized) values.  They query vector is compared
+against these values and, since they are compressed, the comparison is inaccurate.
+
+This parameter can be used to refine the results.  It can improve both improve recall
+and correct the ordering of the nearest results.
+
+To refine results LanceDb will first perform an ANN search to find the nearest
+`limit` * `refine_factor` results.  In other words, if `refine_factor` is 3 and
+`limit` is the default (10) then the first 30 results will be selected.  LanceDb
+then fetches the full, uncompressed, values for these 30 results.  The results are
+then reordered by the true distance and only the nearest 10 are kept.
+
+Note: there is a difference between calling this method with a value of 1 and never
+calling this method at all.  Calling this method with any value will have an impact
+on your search latency.  When you call this method with a `refine_factor` of 1 then
+LanceDb still needs to fetch the full, uncompressed, values so that it can potentially
+reorder the results.
+
+Note: if this method is NOT called then the distances returned in the _distance column
+will be approximate distances based on the comparison of the quantized query vector
+and the quantized result vectors.  This can be considerably different than the true
+distance between the query vector and the actual uncompressed vector.
+
+#### Parameters
+
+• **refineFactor**: `number`
+
+#### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
+### select()
+
+> **select**(`columns`): `this`
+
+Return only the specified columns.
+
+By default a query will return all columns from the table.  However, this can have
+a very significant impact on latency.  LanceDb stores data in a columnar fashion.  This
+means we can finely tune our I/O to select exactly the columns we need.
+
+As a best practice you should always limit queries to the columns that you need.  If you
+pass in an array of column names then only those columns will be returned.
+
+You can also use this method to create new "dynamic" columns based on your existing columns.
+For example, you may not care about "a" or "b" but instead simply want "a + b".  This is often
+seen in the SELECT clause of an SQL query (e.g. `SELECT a+b FROM my_table`).
+
+To create dynamic columns you can pass in a Map<string, string>.  A column will be returned
+for each entry in the map.  The key provides the name of the column.  The value is
+an SQL string used to specify how the column is calculated.
+
+For example, an SQL query might state `SELECT a + b AS combined, c`.  The equivalent
+input to this method would be:
+
+#### Parameters
+
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+new Map([["combined", "a + b"], ["c", "c"]])
+
+Columns will always be returned in the order given, even if that order is different than
+the order used when adding the data.
+
+Note that you can pass in a `Record<string, string>` (e.g. an object literal). This method
+uses `Object.entries` which should preserve the insertion order of the object.  However,
+object insertion order is easy to get wrong and `Map` is more foolproof.
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
+
+***
+
+### toArray()
+
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
+
+Collect the results as an array of objects.
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`any`[]&gt;
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
+
+***
+
+### toArrow()
+
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+Collect the results as an Arrow
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+#### See
+
+ArrowTable.
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
+
+***
+
+### where()
+
+> **where**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+The filter should be supplied as an SQL query string.  For example:
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Example
+
+```ts
+x > 10
+y > 0 AND y < 100
+x > 5 OR y = 'test'
+
+Filtering performance can often be improved by creating a scalar index
+on the filter column(s).
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)

docs/src/js/enumerations/WriteMode.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteMode
+
+# Enumeration: WriteMode
+
+Write mode for writing a table.
+
+## Enumeration Members
+
+### Append
+
+> **Append**: `"Append"`
+
+***
+
+### Create
+
+> **Create**: `"Create"`
+
+***
+
+### Overwrite
+
+> **Overwrite**: `"Overwrite"`

docs/src/js/functions/connect.md

@@ -0,0 +1,82 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / connect
+
+# Function: connect()
+
+## connect(uri, opts)
+
+> **connect**(`uri`, `opts`?): `Promise`<[`Connection`](../classes/Connection.md)>
+
+Connect to a LanceDB instance at the given URI.
+
+Accepted formats:
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (LanceDB cloud)
+
+### Parameters
+
+• **uri**: `string`
+
+The uri of the database. If the database uri starts
+with `db://` then it connects to a remote database.
+
+• **opts?**: `Partial`<[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`>
+
+### Returns
+
+`Promise`<[`Connection`](../classes/Connection.md)>
+
+### See
+
+[ConnectionOptions](../interfaces/ConnectionOptions.md) for more details on the URI format.
+
+### Examples
+
+```ts
+const conn = await connect("/path/to/database");
+```
+
+```ts
+const conn = await connect(
+  "s3://bucket/path/to/database",
+  {storageOptions: {timeout: "60s"}
+});
+```
+
+## connect(opts)
+
+> **connect**(`opts`): `Promise`<[`Connection`](../classes/Connection.md)>
+
+Connect to a LanceDB instance at the given URI.
+
+Accepted formats:
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (LanceDB cloud)
+
+### Parameters
+
+• **opts**: `Partial`<[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`> & `object`
+
+### Returns
+
+`Promise`<[`Connection`](../classes/Connection.md)>
+
+### See
+
+[ConnectionOptions](../interfaces/ConnectionOptions.md) for more details on the URI format.
+
+### Example
+
+```ts
+const conn = await connect({
+  uri: "/path/to/database",
+  storageOptions: {timeout: "60s"}
+});
+```

docs/src/js/functions/makeArrowTable.md

@@ -0,0 +1,114 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / makeArrowTable
+
+# Function: makeArrowTable()
+
+> **makeArrowTable**(`data`, `options`?, `metadata`?): `ArrowTable`
+
+An enhanced version of the makeTable function from Apache Arrow
+that supports nested fields and embeddings columns.
+
+(typically you do not need to call this function.  It will be called automatically
+when creating a table or adding data to it)
+
+This function converts an array of Record<String, any> (row-major JS objects)
+to an Arrow Table (a columnar structure)
+
+Note that it currently does not support nulls.
+
+If a schema is provided then it will be used to determine the resulting array
+types.  Fields will also be reordered to fit the order defined by the schema.
+
+If a schema is not provided then the types will be inferred and the field order
+will be controlled by the order of properties in the first record.  If a type
+is inferred it will always be nullable.
+
+If the input is empty then a schema must be provided to create an empty table.
+
+When a schema is not specified then data types will be inferred.  The inference
+rules are as follows:
+
+ - boolean => Bool
+ - number => Float64
+ - String => Utf8
+ - Buffer => Binary
+ - Record<String, any> => Struct
+ - Array<any> => List
+
+## Parameters
+
+• **data**: `Record`&lt;`string`, `unknown`&gt;[]
+
+• **options?**: `Partial`&lt;[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)&gt;
+
+• **metadata?**: `Map`&lt;`string`, `string`&gt;
+
+## Returns
+
+`ArrowTable`
+
+## Example
+
+import { fromTableToBuffer, makeArrowTable } from "../arrow";
+import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
+
+const schema = new Schema([
+  new Field("a", new Int32()),
+  new Field("b", new Float32()),
+  new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
+ ]);
+ const table = makeArrowTable([
+   { a: 1, b: 2, c: [1, 2, 3] },
+   { a: 4, b: 5, c: [4, 5, 6] },
+   { a: 7, b: 8, c: [7, 8, 9] },
+ ], { schema });
+```
+
+By default it assumes that the column named `vector` is a vector column
+and it will be converted into a fixed size list array of type float32.
+The `vectorColumns` option can be used to support other vector column
+names and data types.
+
+```ts
+
+const schema = new Schema([
+   new Field("a", new Float64()),
+   new Field("b", new Float64()),
+   new Field(
+     "vector",
+     new FixedSizeList(3, new Field("item", new Float32()))
+   ),
+ ]);
+ const table = makeArrowTable([
+   { a: 1, b: 2, vector: [1, 2, 3] },
+   { a: 4, b: 5, vector: [4, 5, 6] },
+   { a: 7, b: 8, vector: [7, 8, 9] },
+ ]);
+ assert.deepEqual(table.schema, schema);
+```
+
+You can specify the vector column types and names using the options as well
+
+```typescript
+
+const schema = new Schema([
+   new Field('a', new Float64()),
+   new Field('b', new Float64()),
+   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ ]);
+const table = makeArrowTable([
+   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ ], {
+   vectorColumns: {
+     vec1: { type: new Float16() },
+     vec2: { type: new Float16() }
+   }
+ }
+assert.deepEqual(table.schema, schema)
+```

docs/src/js/globals.md

@@ -0,0 +1,51 @@
+[**@lancedb/lancedb**](README.md) • **Docs**
+
+***
+
+# @lancedb/lancedb
+
+## Namespaces
+
+- [embedding](namespaces/embedding/README.md)
+
+## Enumerations
+
+- [WriteMode](enumerations/WriteMode.md)
+
+## Classes
+
+- [Connection](classes/Connection.md)
+- [Index](classes/Index.md)
+- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
+- [Query](classes/Query.md)
+- [QueryBase](classes/QueryBase.md)
+- [RecordBatchIterator](classes/RecordBatchIterator.md)
+- [Table](classes/Table.md)
+- [VectorColumnOptions](classes/VectorColumnOptions.md)
+- [VectorQuery](classes/VectorQuery.md)
+
+## Interfaces
+
+- [AddColumnsSql](interfaces/AddColumnsSql.md)
+- [AddDataOptions](interfaces/AddDataOptions.md)
+- [ColumnAlteration](interfaces/ColumnAlteration.md)
+- [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateTableOptions](interfaces/CreateTableOptions.md)
+- [ExecutableQuery](interfaces/ExecutableQuery.md)
+- [IndexConfig](interfaces/IndexConfig.md)
+- [IndexMetadata](interfaces/IndexMetadata.md)
+- [IndexOptions](interfaces/IndexOptions.md)
+- [IndexStatistics](interfaces/IndexStatistics.md)
+- [IvfPqOptions](interfaces/IvfPqOptions.md)
+- [TableNamesOptions](interfaces/TableNamesOptions.md)
+- [UpdateOptions](interfaces/UpdateOptions.md)
+- [WriteOptions](interfaces/WriteOptions.md)
+
+## Type Aliases
+
+- [Data](type-aliases/Data.md)
+
+## Functions
+
+- [connect](functions/connect.md)
+- [makeArrowTable](functions/makeArrowTable.md)

docs/src/js/interfaces/AddColumnsSql.md

@@ -0,0 +1,26 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddColumnsSql
+
+# Interface: AddColumnsSql
+
+A definition of a new column to add to a table.
+
+## Properties
+
+### name
+
+> **name**: `string`
+
+The name of the new column.
+
+***
+
+### valueSql
+
+> **valueSql**: `string`
+
+The values to populate the new column with, as a SQL expression.
+The expression can reference other columns in the table.

docs/src/js/interfaces/AddDataOptions.md

@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddDataOptions
+
+# Interface: AddDataOptions
+
+Options for adding data to a table.
+
+## Properties
+
+### mode
+
+> **mode**: `"append"` \| `"overwrite"`
+
+If "append" (the default) then the new data will be added to the table
+
+If "overwrite" then the new data will replace the existing data in the table.

docs/src/js/interfaces/ColumnAlteration.md

@@ -0,0 +1,40 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ColumnAlteration
+
+# Interface: ColumnAlteration
+
+A definition of a column alteration. The alteration changes the column at
+`path` to have the new name `name`, to be nullable if `nullable` is true,
+and to have the data type `data_type`. At least one of `rename` or `nullable`
+must be provided.
+
+## Properties
+
+### nullable?
+
+> `optional` **nullable**: `boolean`
+
+Set the new nullability. Note that a nullable column cannot be made non-nullable.
+
+***
+
+### path
+
+> **path**: `string`
+
+The path to the column to alter. This is a dot-separated path to the column.
+If it is a top-level column then it is just the name of the column. If it is
+a nested column then it is the path to the column, e.g. "a.b.c" for a column
+`c` nested inside a column `b` nested inside a column `a`.
+
+***
+
+### rename?
+
+> `optional` **rename**: `string`
+
+The new name of the column. If not provided then the name will not be changed.
+This must be distinct from the names of all other columns in the table.

docs/src/js/interfaces/ConnectionOptions.md

@@ -0,0 +1,33 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ConnectionOptions
+
+# Interface: ConnectionOptions
+
+## Properties
+
+### readConsistencyInterval?
+
+> `optional` **readConsistencyInterval**: `number`
+
+(For LanceDB OSS only): The interval, in seconds, at which to check for
+updates to the table from other processes. If None, then consistency is not
+checked. For performance reasons, this is the default. For strong
+consistency, set this to zero seconds. Then every read will check for
+updates from other processes. As a compromise, you can set this to a
+non-zero value for eventual consistency. If more than that interval
+has passed since the last check, then the table will be checked for updates.
+Note: this consistency only applies to read operations. Write operations are
+always consistent.
+
+***
+
+### storageOptions?
+
+> `optional` **storageOptions**: `Record`<`string`, `string`>
+
+(For LanceDB OSS only): configuration for object storage.
+
+The available options are described at https://lancedb.github.io/lancedb/guides/storage/

docs/src/js/interfaces/CreateTableOptions.md

@@ -0,0 +1,65 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateTableOptions
+
+# Interface: CreateTableOptions
+
+## Properties
+
+### embeddingFunction?
+
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
+
+***
+
+### existOk
+
+> **existOk**: `boolean`
+
+If this is true and the table already exists and the mode is "create"
+then no error will be raised.
+
+***
+
+### mode
+
+> **mode**: `"overwrite"` \| `"create"`
+
+The mode to use when creating the table.
+
+If this is set to "create" and the table already exists then either
+an error will be thrown or, if existOk is true, then nothing will
+happen.  Any provided data will be ignored.
+
+If this is set to "overwrite" then any existing table will be replaced.
+
+***
+
+### schema?
+
+> `optional` **schema**: `SchemaLike`
+
+***
+
+### storageOptions?
+
+> `optional` **storageOptions**: `Record`<`string`, `string`>
+
+Configuration for object storage.
+
+Options already set on the connection will be inherited by the table,
+but can be overridden here.
+
+The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+
+***
+
+### useLegacyFormat?
+
+> `optional` **useLegacyFormat**: `boolean`
+
+If true then data files will be written with the legacy format
+
+The default is true while the new format is in beta

docs/src/js/interfaces/ExecutableQuery.md

@@ -0,0 +1,11 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ExecutableQuery
+
+# Interface: ExecutableQuery
+
+An interface for a query that can be executed
+
+Supported by all query types

docs/src/js/interfaces/IndexConfig.md

@@ -0,0 +1,36 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexConfig
+
+# Interface: IndexConfig
+
+A description of an index currently configured on a column
+
+## Properties
+
+### columns
+
+> **columns**: `string`[]
+
+The columns in the index
+
+Currently this is always an array of size 1. In the future there may
+be more columns to represent composite indices.
+
+***
+
+### indexType
+
+> **indexType**: `string`
+
+The type of the index
+
+***
+
+### name
+
+> **name**: `string`
+
+The name of the index

docs/src/js/interfaces/IndexMetadata.md

@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexMetadata
+
+# Interface: IndexMetadata
+
+## Properties
+
+### indexType?
+
+> `optional` **indexType**: `string`
+
+***
+
+### metricType?
+
+> `optional` **metricType**: `string`

docs/src/js/interfaces/IndexOptions.md

@@ -0,0 +1,37 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexOptions
+
+# Interface: IndexOptions
+
+## Properties
+
+### config?
+
+> `optional` **config**: [`Index`](../classes/Index.md)
+
+Advanced index configuration
+
+This option allows you to specify a specfic index to create and also
+allows you to pass in configuration for training the index.
+
+See the static methods on Index for details on the various index types.
+
+If this is not supplied then column data type(s) and column statistics
+will be used to determine the most useful kind of index to create.
+
+***
+
+### replace?
+
+> `optional` **replace**: `boolean`
+
+Whether to replace the existing index
+
+If this is false, and another index already exists on the same columns
+and the same name, then an error will be returned.  This is true even if
+that index is out of date.
+
+The default is true

docs/src/js/interfaces/IndexStatistics.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexStatistics
+
+# Interface: IndexStatistics
+
+## Properties
+
+### indexType?
+
+> `optional` **indexType**: `string`
+
+The type of the index
+
+***
+
+### indices
+
+> **indices**: [`IndexMetadata`](IndexMetadata.md)[]
+
+The metadata for each index
+
+***
+
+### numIndexedRows
+
+> **numIndexedRows**: `number`
+
+The number of rows indexed by the index
+
+***
+
+### numUnindexedRows
+
+> **numUnindexedRows**: `number`
+
+The number of rows not indexed

docs/src/js/interfaces/IvfPqOptions.md

@@ -0,0 +1,118 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IvfPqOptions
+
+# Interface: IvfPqOptions
+
+Options to create an `IVF_PQ` index
+
+## Properties
+
+### distanceType?
+
+> `optional` **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
+
+Distance type to use to build the index.
+
+Default value is "l2".
+
+This is used when training the index to calculate the IVF partitions
+(vectors are grouped in partitions with similar vectors according to this
+distance type) and to calculate a subvector's code during quantization.
+
+The distance type used to train an index MUST match the distance type used
+to search the index.  Failure to do so will yield inaccurate results.
+
+The following distance types are available:
+
+"l2" - Euclidean distance. This is a very common distance metric that
+accounts for both magnitude and direction when determining the distance
+between vectors. L2 distance has a range of [0, ∞).
+
+"cosine" - Cosine distance.  Cosine distance is a distance metric
+calculated from the cosine similarity between two vectors. Cosine
+similarity is a measure of similarity between two non-zero vectors of an
+inner product space. It is defined to equal the cosine of the angle
+between them.  Unlike L2, the cosine distance is not affected by the
+magnitude of the vectors.  Cosine distance has a range of [0, 2].
+
+Note: the cosine distance is undefined when one (or both) of the vectors
+are all zeros (there is no direction).  These vectors are invalid and may
+never be returned from a vector search.
+
+"dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+L2 norm is 1), then dot distance is equivalent to the cosine distance.
+
+***
+
+### maxIterations?
+
+> `optional` **maxIterations**: `number`
+
+Max iteration to train IVF kmeans.
+
+When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
+controls how many iterations of kmeans to run.
+
+Increasing this might improve the quality of the index but in most cases these extra
+iterations have diminishing returns.
+
+The default value is 50.
+
+***
+
+### numPartitions?
+
+> `optional` **numPartitions**: `number`
+
+The number of IVF partitions to create.
+
+This value should generally scale with the number of rows in the dataset.
+By default the number of partitions is the square root of the number of
+rows.
+
+If this value is too large then the first part of the search (picking the
+right partition) will be slow.  If this value is too small then the second
+part of the search (searching within a partition) will be slow.
+
+***
+
+### numSubVectors?
+
+> `optional` **numSubVectors**: `number`
+
+Number of sub-vectors of PQ.
+
+This value controls how much the vector is compressed during the quantization step.
+The more sub vectors there are the less the vector is compressed.  The default is
+the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+by 16 we use the dimension divded by 8.
+
+The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+us to use efficient SIMD instructions.
+
+If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+will likely result in poor performance.
+
+***
+
+### sampleRate?
+
+> `optional` **sampleRate**: `number`
+
+The number of vectors, per partition, to sample when training IVF kmeans.
+
+When an IVF PQ index is trained, we need to calculate partitions.  These are groups
+of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+
+Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+random sample of the data.  This parameter controls the size of the sample.  The total
+number of vectors used to train the index is `sample_rate * num_partitions`.
+
+Increasing this value might improve the quality of the index but in most cases the
+default should be sufficient.
+
+The default value is 256.

docs/src/js/interfaces/TableNamesOptions.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TableNamesOptions
+
+# Interface: TableNamesOptions
+
+## Properties
+
+### limit?
+
+> `optional` **limit**: `number`
+
+An optional limit to the number of results to return.
+
+***
+
+### startAfter?
+
+> `optional` **startAfter**: `string`
+
+If present, only return names that come lexicographically after the
+supplied value.
+
+This can be combined with limit to implement pagination by setting this to
+the last table name from the previous page.