[python] Bump version: 0.1.14 → 0.1.15

Closes #359

TODO:
 * [x] test in a sample of Linux distro docker containers

.bumpversion.cfg

@@ -0,0 +1,12 @@
+[bumpversion]
+current_version = 0.1.18
+commit = True
+message = Bump version: {current_version} → {new_version}
+tag = True
+tag_name = v{new_version}
+
+[bumpversion:file:node/package.json]
+
+[bumpversion:file:rust/ffi/node/Cargo.toml]
+
+[bumpversion:file:rust/vectordb/Cargo.toml]

.github/workflows/cargo-publish.yml

@@ -0,0 +1,29 @@
+name: Cargo Publish
+
+on:
+  release:
+    types: [ published ]
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Publish the package
+        run: |
+          cargo publish -p vectordb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/docs.yml

@@ -39,6 +39,28 @@ jobs:
         run: |
           python -m pip install -e .
           python -m pip install -r ../docs/requirements.txt
+      - name: Set up node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
+      - name: Install node dependencies
+        working-directory: node
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build node
+        working-directory: node
+        run: |
+          npm ci
+          npm run build
+          npm run tsc
+      - name: Create markdown files
+        working-directory: node
+        run: |
+          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
       - name: Build docs
         run: |
           PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
@@ -50,4 +72,4 @@ jobs:
           path: "docs/site"
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/docs_test.yml

@@ -0,0 +1,93 @@
+name: Documentation Code Testing
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+  pull_request:
+    paths:
+      - docs/**
+      - .github/workflows/docs_test.yml
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  test-python:
+    name: Test doc python code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-minor-version: [ "11" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.${{ matrix.python-minor-version }}
+        cache: "pip"
+        cache-dependency-path: "docs/test/requirements.txt"
+    - name: Build Python
+      working-directory: docs/test
+      run:
+        python -m pip install -r requirements.txt
+    - name: Create test files
+      run: |
+        cd docs/test
+        python md_testing.py
+    - name: Test
+      run: |
+        cd docs/test/python
+        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
+  test-node:
+    name: Test doc nodejs code
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ "18" ]
+        os: ["ubuntu-22.04"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Install dependecies needed for ubuntu
+      if: ${{ matrix.os == 'ubuntu-22.04' }}
+      run: |
+        sudo apt install -y protobuf-compiler libssl-dev
+    - name: Install node dependencies
+      run: |
+        cd docs/test
+        npm install 
+    - name: Rust cache
+      uses: swatinem/rust-cache@v2
+    - name: Install LanceDB
+      run: |
+        cd docs/test/node_modules/vectordb 
+        npm ci
+        npm run build-release
+        npm run tsc
+    - name: Create test files
+      run: |
+        cd docs/test
+        node md_testing.js
+    - name: Test
+      run: |
+        cd docs/test/node
+        for d in *; do cd "$d"; echo "$d".js; node "$d".js; cd ..; done

.github/workflows/make-release-commit.yml

@@ -0,0 +1,59 @@
+name: Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out main
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - name: Set git configs for bumpversion
+        shell: bash
+        run: |
+          git config user.name 'Lance Release'
+          git config user.email 'lance-dev@lancedb.com'
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Bump version, create tag and commit
+        run: |
+          pip install bump2version
+          bumpversion --verbose ${{ inputs.part }}
+      - name: Push new version and tag
+        if: ${{ inputs.dry_run }} == "false"
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          branch: main
+          tags: true
+      - uses: ./.github/workflows/update_package_lock
+        if: ${{ inputs.dry_run }} == "false"
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+

.github/workflows/node.yml

@@ -67,8 +67,12 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
     - name: Test
       run: npm run test
   macos:
@@ -94,8 +98,12 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -0,0 +1,163 @@
+name: NPM Publish
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  node:
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: node-package
+          path: |
+            node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install system dependencies
+        run: brew install protobuf
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Install rustup target
+        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+        run: rustup target add aarch64-apple-darwin
+      - name: Build MacOS native node modules
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+      - name: Upload Darwin Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-darwin
+          path: |
+            node/dist/lancedb-vectordb-darwin*.tgz
+
+  node-linux:
+    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    runs-on: ${{ matrix.config.runner }}
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - arch: x86_64
+            runner: ubuntu-latest
+          - arch: aarch64
+            runner: buildjet-4vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Linux Artifacts
+        run: |
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-linux
+          path: |
+            node/dist/lancedb-vectordb-linux*.tgz
+
+  node-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-windows
+          path: |
+            node/dist/lancedb-vectordb-win32*.tgz
+
+  release:
+    needs: [node, node-macos, node-linux, node-windows]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Display structure of downloaded files
+        run: ls -R
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: 'https://registry.npmjs.org'
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: |
+          mv */*.tgz .
+          for filename in *.tgz; do
+            npm publish $filename
+          done
+
+  update-package-lock:
+    needs: [release]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/pypi-publish.yml

@@ -0,0 +1,31 @@
+name: PyPI Publish
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the python-make-release action
+    if: startsWith(github.ref, 'refs/tags/python-v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+      - name: Build distribution
+        run: |
+          ls -la
+          pip install wheel setuptools --upgrade
+          python setup.py sdist bdist_wheel
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@v1.8.5
+        with:
+          password: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
+          packages-dir: python/dist

.github/workflows/python-make-release-commit.yml

@@ -0,0 +1,56 @@
+name: Python - Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Dry run (create the local commit/tags but do not push it)'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Set git configs for bumpversion
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Bump version, create tag and commit
+      working-directory: python
+      run: |
+        pip install bump2version
+        bumpversion --verbose ${{ inputs.part }}
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+        branch: main
+        tags: true
+

.github/workflows/python.yml

@@ -32,9 +32,15 @@ jobs:
       run: |
         pip install -e .
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest pytest-mock black isort
+    - name: Black
+      run: black --check --diff --no-color --quiet .
+    - name: isort
+      run: isort --check --diff --quiet .
     - name: Run tests
       run: pytest -x -v --durations=30 tests
+    - name: doctest
+      run: pytest --doctest-modules lancedb
   mac:
     timeout-minutes: 30
     runs-on: "macos-12"
@@ -55,6 +61,8 @@ jobs:
       run: |
         pip install -e .
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock
+        pip install pytest pytest-mock black
+    - name: Black
+      run: black --check --diff --no-color --quiet .
     - name: Run tests
-      run: pytest -x -v --durations=30 tests
+      run: pytest -x -v --durations=30 tests

.github/workflows/rust.yml

@@ -0,0 +1,89 @@
+name: Rust
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - Cargo.toml
+      - rust/**
+      - .github/workflows/rust.yml
+
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+
+jobs:
+  linux:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+          workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Build
+      run: cargo build --all-features
+    - name: Run tests
+      run: cargo test --all-features
+  macos:
+    runs-on: macos-12
+    timeout-minutes: 30
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+      - uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+            lfs: true
+      - name: CPU features
+        run: sysctl -a | grep cpu
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install dependencies
+        run: brew install protobuf
+      - name: Build
+        run: cargo build --all-features
+      - name: Run tests
+        run: cargo test --all-features
+  windows:
+    runs-on: windows-2022
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Run tests
+        run: |
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
+          cargo build
+          cargo test

.github/workflows/update_package_lock/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock
+description: "Update node's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./node
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run.yml

@@ -0,0 +1,19 @@
+name: Update package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.gitignore

@@ -3,6 +3,9 @@
 *.egg-info
 **/__pycache__
 .DS_Store
+venv
+
+.vscode
 
 rust/target
 rust/Cargo.lock
@@ -30,3 +33,4 @@ node/examples/**/dist
 ## Rust
 target
 
+Cargo.lock

.pre-commit-config.yaml

@@ -8,4 +8,14 @@ repos:
 -   repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
-    -   id: black
+    -   id: black
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.277
+    hooks:
+    - id: ruff
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      name: isort (python)

Cargo.toml

@@ -4,3 +4,13 @@ members = [
     "rust/ffi/node"
 ]
 resolver = "2"
+
+[workspace.dependencies]
+lance = "=0.5.9"
+arrow-array = "42.0"
+arrow-data = "42.0"
+arrow-schema = "42.0"
+arrow-ipc = "42.0"
+half = { "version" = "=2.2.1", default-features = false }
+object_store = "0.6.1"
+

README.md

@@ -65,7 +65,7 @@ pip install lancedb
 ```python
 import lancedb
 
-uri = "/tmp/lancedb"
+uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 table = db.create_table("my_table",
                          data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -75,4 +75,4 @@ result = table.search([100, 100]).limit(2).to_df()
 
 ## Blogs, Tutorials & Videos
 * 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
-* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>
+* 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

ci/build_linux_artifacts.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+ARCH=${1:-x86_64}
+
+# We pass down the current user so that when we later mount the local files 
+# into the container, the files are accessible by the current user.
+pushd ci/manylinux_node
+docker build \
+    -t lancedb-node-manylinux \
+    --build-arg="ARCH=$ARCH" \
+    --build-arg="DOCKER_USER=$(id -u)" \
+    --progress=plain \
+    .
+popd
+
+docker run \
+    -v $(pwd):/io -w /io \
+    lancedb-node-manylinux \
+    bash ci/manylinux_node/build.sh $ARCH

ci/build_macos_artifacts.sh

@@ -0,0 +1,33 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./ci/build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/ffi/node
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd node
+    echo "Building node library for $1"
+    npm run build-release -- --target $1
+    npm run pack-build -- --target $1
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (node binaries).
+# Usage:  .\ci\build_windows_artifacts.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/ffi/node"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "node"
+    Write-Host "Building node library for $target"
+    npm run build-release -- --target $target
+    npm run pack-build -- --target $target
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/manylinux_node/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an 
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be 
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_node/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else 
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd node
+npm ci
+npm run build-release
+npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_node/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_node/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

docs/mkdocs.yml

@@ -6,22 +6,39 @@ docs_dir: src
 theme:
   name: "material"
   logo: assets/logo.png
+  favicon: assets/logo.png
   features:
     - content.code.copy
     - content.tabs.link
   icon:
     repo: fontawesome/brands/github
+  custom_dir: overrides
 
 plugins:
 - search
+- autorefs
 - mkdocstrings:
     handlers:
       python:
         paths: [../python]
+        selection:
+          docstring_style: numpy
+        rendering:
+          heading_level: 4
+          show_source: false
+          show_symbol_type_in_heading: true
+          show_signature_annotations: true
+          show_root_heading: true
+          members_order: source
+        import:
+          # for cross references
+          - https://arrow.apache.org/docs/objects.inv
+          - https://pandas.pydata.org/docs/objects.inv
 - mkdocs-jupyter
 
 markdown_extensions:
 - admonition
+- footnotes
 - pymdownx.superfences
 - pymdownx.details
 - pymdownx.highlight:
@@ -33,19 +50,31 @@ markdown_extensions:
 - pymdownx.superfences
 - pymdownx.tabbed:
     alternate_style: true
+- md_in_html
 
 nav:
 - Home: index.md
 - Basics: basic.md
 - Embeddings: embedding.md
 - Python full-text search: fts.md
-- Python integrations: integrations.md
+- Python integrations:
+  - Pandas and PyArrow: python/arrow.md
+  - DuckDB: python/duckdb.md
+  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+  - Pydantic: python/pydantic.md
 - Python examples:
-  - YouTube Transcript Search using OpenAI: notebooks/youtube_transcript_search.ipynb
+  - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
   - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+  - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+  - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+- Javascript examples:
+  - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - References:
   - Vector Search: search.md
+  - SQL filters: sql.md
   - Indexing: ann_indexes.md
 - API references:
   - Python API: python/python.md

docs/overrides/partials/header.html

@@ -0,0 +1,176 @@
+<!--
+  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to
+  deal in the Software without restriction, including without limitation the
+  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+  sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+  IN THE SOFTWARE.
+-->
+
+{% set class = "md-header" %}
+{% if "navigation.tabs.sticky" in features %}
+  {% set class = class ~ " md-header--shadow md-header--lifted" %}
+{% elif "navigation.tabs" not in features %}
+  {% set class = class ~ " md-header--shadow" %}
+{% endif %}
+
+<!-- Header -->
+<header class="{{ class }}" data-md-component="header">
+  <nav
+    class="md-header__inner md-grid"
+    aria-label="{{ lang.t('header') }}"
+  >
+
+    <!-- Link to home -->
+    <a
+      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
+      title="{{ config.site_name | e }}"
+      class="md-header__button md-logo"
+      aria-label="{{ config.site_name }}"
+      data-md-component="logo"
+    >
+      {% include "partials/logo.html" %}
+    </a>
+
+    <!-- Button to open drawer -->
+    <label class="md-header__button md-icon" for="__drawer">
+      {% include ".icons/material/menu" ~ ".svg" %}
+    </label>
+
+    <!-- Header title -->
+    <div class="md-header__title" style="width: auto !important;" data-md-component="header-title">
+      <div class="md-header__ellipsis">
+        <div class="md-header__topic">
+          <span class="md-ellipsis">
+            {{ config.site_name }}
+          </span>
+        </div>
+        <div class="md-header__topic" data-md-component="header-topic">
+          <span class="md-ellipsis">
+            {% if page.meta and page.meta.title %}
+              {{ page.meta.title }}
+            {% else %}
+              {{ page.title }}
+            {% endif %}
+          </span>
+        </div>
+      </div>
+    </div>
+
+    <!-- Color palette -->
+    {% if config.theme.palette %}
+      {% if not config.theme.palette is mapping %}
+        <form class="md-header__option" data-md-component="palette">
+          {% for option in config.theme.palette %}
+            {% set scheme  = option.scheme  | d("default", true) %}
+            {% set primary = option.primary | d("indigo", true) %}
+            {% set accent  = option.accent  | d("indigo", true) %}
+            <input
+              class="md-option"
+              data-md-color-media="{{ option.media }}"
+              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
+              data-md-color-primary="{{ primary | replace(' ', '-') }}"
+              data-md-color-accent="{{ accent | replace(' ', '-') }}"
+              {% if option.toggle %}
+                aria-label="{{ option.toggle.name }}"
+              {% else  %}
+                aria-hidden="true"
+              {% endif %}
+              type="radio"
+              name="__palette"
+              id="__palette_{{ loop.index }}"
+            />
+            {% if option.toggle %}
+              <label
+                class="md-header__button md-icon"
+                title="{{ option.toggle.name }}"
+                for="__palette_{{ loop.index0 or loop.length }}"
+                hidden
+              >
+                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
+              </label>
+            {% endif %}
+          {% endfor %}
+        </form>
+      {% endif %}
+    {% endif %}
+
+    <!-- Site language selector -->
+    {% if config.extra.alternate %}
+      <div class="md-header__option">
+        <div class="md-select">
+          {% set icon = config.theme.icon.alternate or "material/translate" %}
+          <button
+            class="md-header__button md-icon"
+            aria-label="{{ lang.t('select.language') }}"
+          >
+            {% include ".icons/" ~ icon ~ ".svg" %}
+          </button>
+          <div class="md-select__inner">
+            <ul class="md-select__list">
+              {% for alt in config.extra.alternate %}
+                <li class="md-select__item">
+                  <a
+                    href="{{ alt.link | url }}"
+                    hreflang="{{ alt.lang }}"
+                    class="md-select__link"
+                  >
+                    {{ alt.name }}
+                  </a>
+                </li>
+              {% endfor %}
+            </ul>
+          </div>
+        </div>
+      </div>
+    {% endif %}
+
+    <!-- Button to open search modal -->
+    {% if "material/search" in config.plugins %}
+      <label class="md-header__button md-icon" for="__search">
+        {% include ".icons/material/magnify.svg" %}
+      </label>
+
+      <!-- Search interface -->
+      {% include "partials/search.html" %}
+    {% endif %}
+
+    <div style="margin-left: 10px; margin-right: 5px;">
+        <a href="https://discord.com/invite/zMM32dvNtd" target="_blank" rel="noopener noreferrer">
+            <svg fill="#FFFFFF" xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 50 50" width="25px" height="25px"><path d="M 41.625 10.769531 C 37.644531 7.566406 31.347656 7.023438 31.078125 7.003906 C 30.660156 6.96875 30.261719 7.203125 30.089844 7.589844 C 30.074219 7.613281 29.9375 7.929688 29.785156 8.421875 C 32.417969 8.867188 35.652344 9.761719 38.578125 11.578125 C 39.046875 11.867188 39.191406 12.484375 38.902344 12.953125 C 38.710938 13.261719 38.386719 13.429688 38.050781 13.429688 C 37.871094 13.429688 37.6875 13.378906 37.523438 13.277344 C 32.492188 10.15625 26.210938 10 25 10 C 23.789063 10 17.503906 10.15625 12.476563 13.277344 C 12.007813 13.570313 11.390625 13.425781 11.101563 12.957031 C 10.808594 12.484375 10.953125 11.871094 11.421875 11.578125 C 14.347656 9.765625 17.582031 8.867188 20.214844 8.425781 C 20.0625 7.929688 19.925781 7.617188 19.914063 7.589844 C 19.738281 7.203125 19.34375 6.960938 18.921875 7.003906 C 18.652344 7.023438 12.355469 7.566406 8.320313 10.8125 C 6.214844 12.761719 2 24.152344 2 34 C 2 34.175781 2.046875 34.34375 2.132813 34.496094 C 5.039063 39.605469 12.972656 40.941406 14.78125 41 C 14.789063 41 14.800781 41 14.8125 41 C 15.132813 41 15.433594 40.847656 15.621094 40.589844 L 17.449219 38.074219 C 12.515625 36.800781 9.996094 34.636719 9.851563 34.507813 C 9.4375 34.144531 9.398438 33.511719 9.765625 33.097656 C 10.128906 32.683594 10.761719 32.644531 11.175781 33.007813 C 11.234375 33.0625 15.875 37 25 37 C 34.140625 37 38.78125 33.046875 38.828125 33.007813 C 39.242188 32.648438 39.871094 32.683594 40.238281 33.101563 C 40.601563 33.515625 40.5625 34.144531 40.148438 34.507813 C 40.003906 34.636719 37.484375 36.800781 32.550781 38.074219 L 34.378906 40.589844 C 34.566406 40.847656 34.867188 41 35.1875 41 C 35.199219 41 35.210938 41 35.21875 41 C 37.027344 40.941406 44.960938 39.605469 47.867188 34.496094 C 47.953125 34.34375 48 34.175781 48 34 C 48 24.152344 43.785156 12.761719 41.625 10.769531 Z M 18.5 30 C 16.566406 30 15 28.210938 15 26 C 15 23.789063 16.566406 22 18.5 22 C 20.433594 22 22 23.789063 22 26 C 22 28.210938 20.433594 30 18.5 30 Z M 31.5 30 C 29.566406 30 28 28.210938 28 26 C 28 23.789063 29.566406 22 31.5 22 C 33.433594 22 35 23.789063 35 26 C 35 28.210938 33.433594 30 31.5 30 Z"/></svg>
+        </a>
+    </div>
+    <div style="margin-left: 5px; margin-right: 5px;">
+        <a href="https://twitter.com/lancedb" target="_blank" rel="noopener noreferrer">
+            <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0,0,256,256" width="25px" height="25px" fill-rule="nonzero"><g fill-opacity="0" fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><path d="M0,256v-256h256v256z" id="bgRectangle"></path></g><g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(4,4)"><path d="M57,17.114c-1.32,1.973 -2.991,3.707 -4.916,5.097c0.018,0.423 0.028,0.847 0.028,1.274c0,13.013 -9.902,28.018 -28.016,28.018c-5.562,0 -12.81,-1.948 -15.095,-4.423c0.772,0.092 1.556,0.138 2.35,0.138c4.615,0 8.861,-1.575 12.23,-4.216c-4.309,-0.079 -7.946,-2.928 -9.199,-6.84c1.96,0.308 4.447,-0.17 4.447,-0.17c0,0 -7.7,-1.322 -7.899,-9.779c2.226,1.291 4.46,1.231 4.46,1.231c0,0 -4.441,-2.734 -4.379,-8.195c0.037,-3.221 1.331,-4.953 1.331,-4.953c8.414,10.361 20.298,10.29 20.298,10.29c0,0 -0.255,-1.471 -0.255,-2.243c0,-5.437 4.408,-9.847 9.847,-9.847c2.832,0 5.391,1.196 7.187,3.111c2.245,-0.443 4.353,-1.263 6.255,-2.391c-0.859,3.44 -4.329,5.448 -4.329,5.448c0,0 2.969,-0.329 5.655,-1.55z"></path></g></g></svg>
+        </a>
+    </div>
+
+    <!-- Repository information -->
+    {% if config.repo_url %}
+      <div class="md-header__source" style="margin-left: -5px !important;">
+        {% include "partials/source.html" %}
+      </div>
+    {% endif %}
+  </nav>
+
+  <!-- Navigation tabs (sticky) -->
+  {% if "navigation.tabs.sticky" in features %}
+    {% if "navigation.tabs" in features %}
+      {% include "partials/tabs.html" %}
+    {% endif %}
+  {% endif %}
+</header>

docs/src/ann_indexes.md

@@ -1,7 +1,7 @@
 # ANN (Approximate Nearest Neighbor) Indexes
 
 You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search.
+Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
 Currently, LanceDB does *not* automatically create the ANN index.
@@ -10,7 +10,18 @@ If you can live with <100ms latency, skipping index creation is a simpler workfl
 
 In the future we will look to automatically create and configure the ANN index.
 
-## Creating an ANN Index
+## Types of Index
+
+Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+
+* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+   and then use **Product Quantization** to compress vectors in each partition.
+* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+   represent the nearest neighbors of each vector.
+
+## Creating an IVF_PQ Index
+
+Lance supports `IVF_PQ` index type by default.
 
 === "Python"
      Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
@@ -23,7 +34,7 @@ In the future we will look to automatically create and configure the ANN index.
 
      # Create 10,000 sample vectors
      data = [{"vector": row, "item": f"item {i}"}
-        for i, row in enumerate(np.random.random((10_000, 768)).astype('float32'))]
+        for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
 
      # Add the vectors to a table
      tbl = db.create_table("my_vectors", data=data)
@@ -41,19 +52,22 @@ In the future we will look to automatically create and configure the ANN index.
      for (let i = 0; i < 10_000; i++) {
          data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
      }
-     const table = await db.createTable('vectors', data)
-     await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
+     const table = await db.createTable('my_vectors', data)
+     await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
      ```
 
-Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
-creation by providing the following parameters:
+- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+We also support "cosine" and "dot" distance as well.
+- **num_partitions** (default: 256): The number of partitions of the index.
+- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
+a single PQ code.
+
+<figure markdown>
+  ![IVF PQ](./assets/ivf_pq.png)
+  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
+</figure>
 
-- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support "cosine" distance.
-- **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
-with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
-A higher number leads to faster queries, but it makes index generation slower.
-- **num_sub_vectors** (default: 96): The number of subvectors (M) that will be created during Product Quantization (PQ). A larger number makes
-search more accurate, but also makes the index larger and slower to build.
 
 ## Querying an ANN Index
 
@@ -67,18 +81,19 @@ There are a couple of parameters that can be used to fine-tune the search:
   e.g., for 1M vectors divided up into 256 partitions, nprobes should be set to ~20-40.<br/>
   Note: nprobes is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 - **refine_factor** (default: None): Refine the results by reading extra elements and re-ranking them in memory.<br/>
-  A higher number makes search more accurate but also slower. If you find the recall is less than idea, try refine_factor=10 to start.<br/>
+  A higher number makes search more accurate but also slower. If you find the recall is less than ideal, try refine_factor=10 to start.<br/>
   e.g., for 1M vectors divided into 256 partitions, if you're looking for top 20, then refine_factor=200 reranks the whole partition.<br/>
   Note: refine_factor is only applicable if an ANN index is present. If specified on a table without an ANN index, it is ignored.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))) \
+     tbl.search(np.random.random((1536))) \
          .limit(2) \
          .nprobes(20) \
          .refine_factor(10) \
          .to_df()
-
+     ```
+     ```
                                               vector       item       score
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
@@ -86,8 +101,8 @@ There are a couple of parameters that can be used to fine-tune the search:
 
 === "Javascript"
      ```javascript
-     const results = await table
-         .search(Array(768).fill(1.2))
+     const results_1 = await table
+         .search(Array(1536).fill(1.2))
          .limit(2)
          .nprobes(20)
          .refineFactor(10)
@@ -104,14 +119,14 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))).where("item != 'item 1141'").to_df()
+     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_df()
      ```
 
 === "Javascript"
      ```javascript
-     const results = await table
+     const results_2 = await table
          .search(Array(1536).fill(1.2))
-         .where("item != 'item 1141'")
+         .where("id != '1141'")
          .execute()
      ```
 
@@ -121,7 +136,9 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
      ```python
-     tbl.search(np.random.random((768))).select(["vector"]).to_df()
+     tbl.search(np.random.random((1536))).select(["vector"]).to_df()
+     ```
+     ```
      vector      score
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
@@ -130,8 +147,36 @@ You can select the columns returned by the query using a select clause.
 
 === "Javascript"
      ```javascript
-     const results = await table
+     const results_3 = await table
          .search(Array(1536).fill(1.2))
          .select(["id"])
          .execute()
      ```
+
+## FAQ
+
+### When is it necessary to create an ANN vector index.
+
+`LanceDB` has manually tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
+For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how many memory will it take.
+
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
+Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
+
+`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
+Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
+On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
+
+`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
+Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
+less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
+more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.

docs/src/basic.md

@@ -23,7 +23,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 === "Python"
       ```python
       import lancedb
-      uri = "~/.lancedb"
+      uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       ```
 
@@ -35,7 +35,7 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       ```javascript
       const lancedb = require("vectordb");
 
-      const uri = "~./lancedb";
+      const uri = "data/sample-lancedb";
       const db = await lancedb.connect(uri);
       ```
       
@@ -102,7 +102,7 @@ Once created, you can open a table using the following code:
       If you forget the name of your table, you can always get a listing of all table names:
 
       ```javascript
-      console.log(db.tableNames());
+      console.log(await db.tableNames());
       ```
 
 ## How to add data to a table
@@ -118,10 +118,39 @@ After a table has been created, you can always add more data to it using
 
 === "Javascript"
       ```javascript
-      await tbl.add([vector: [1.3, 1.4], item: "fizz", price: 100.0},
+      await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
               {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
       ```
 
+## How to delete rows from a table
+
+Use the `delete()` method on tables to delete rows from a table. To choose
+which rows to delete, provide a filter that matches on the metadata columns.
+This can delete any number of rows that match the filter.
+
+=== "Python"
+      ```python
+      tbl.delete('item = "fizz"')
+      ```
+
+=== "Javascript"
+      ```javascript
+      await tbl.delete('item = "fizz"')
+      ```
+
+The deletion predicate is a SQL expression that supports the same expressions
+as the `where()` clause on a search. They can be as simple or complex as needed.
+To see what expressions are supported, see the [SQL filters](sql.md) section.
+
+
+=== "Python"
+
+      Read more: [lancedb.table.Table.delete][]
+
+=== "Javascript"
+
+      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:

docs/src/embedding.md

@@ -45,8 +45,8 @@ You can also use an external API like OpenAI to generate embeddings
         assert len(openai.Model.list()["data"]) > 0
 
         def embed_func(c):
-        rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-        return [record["embedding"] for record in rs["data"]]
+            rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
+            return [record["embedding"] for record in rs["data"]]
       ```
 
 === "Javascript"
@@ -98,7 +98,7 @@ You can also use an external API like OpenAI to generate embeddings
      embededings for your data.
 
       ```javascript
-      const db = await lancedb.connect("/tmp/lancedb");
+      const db = await lancedb.connect("data/sample-lancedb");
       const data = [
         { text: 'pepperoni'  },
         { text: 'pineapple' }
@@ -126,7 +126,7 @@ belong in the same latent space and your results will be nonsensical.
 === "Javascript"
      ```javascript
       const results = await table
-        .search('What's the best pizza topping?')
+        .search("What's the best pizza topping?")
         .limit(10)
         .execute()
      ```

docs/src/examples/modal_langchain.py

@@ -1,27 +1,22 @@
-import sys
-from modal import Secret, Stub, Image, web_endpoint
-import lancedb
-import re
 import pickle
-import requests
+import re
+import sys
 import zipfile
 from pathlib import Path
 
+import requests
+from langchain.chains import RetrievalQA
 from langchain.document_loaders import UnstructuredHTMLLoader
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import LanceDB
-from langchain.llms import OpenAI
-from langchain.chains import RetrievalQA
+from modal import Image, Secret, Stub, web_endpoint
+
+import lancedb
 
 lancedb_image = Image.debian_slim().pip_install(
-    "lancedb",
-    "langchain",
-    "openai",
-    "pandas",
-    "tiktoken",
-    "unstructured",
-    "tabulate"
+    "lancedb", "langchain", "openai", "pandas", "tiktoken", "unstructured", "tabulate"
 )
 
 stub = Stub(
@@ -34,21 +29,26 @@ docsearch = None
 docs_path = Path("docs.pkl")
 db_path = Path("lancedb")
 
+
 def get_document_title(document):
     m = str(document.metadata["source"])
     title = re.findall("pandas.documentation(.*).html", m)
     if title[0] is not None:
-        return(title[0])
-    return ''
+        return title[0]
+    return ""
+
 
 def download_docs():
-    pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
+    pandas_docs = requests.get(
+        "https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"
+    )
     with open(Path("pandas.documentation.zip"), "wb") as f:
         f.write(pandas_docs.content)
 
     file = zipfile.ZipFile(Path("pandas.documentation.zip"))
     file.extractall(path=Path("pandas_docs"))
 
+
 def store_docs():
     docs = []
 
@@ -74,25 +74,34 @@ def store_docs():
 
     return docs
 
+
 def qanda_langchain(query):
     download_docs()
     docs = store_docs()
 
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
-    )
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
     documents = text_splitter.split_documents(docs)
     embeddings = OpenAIEmbeddings()
 
-    db = lancedb.connect(db_path) 
-    table = db.create_table("pandas_docs", data=[
-        {"vector": embeddings.embed_query("Hello World"), "text": "Hello World", "id": "1"}
-    ], mode="overwrite")
+    db = lancedb.connect(db_path)
+    table = db.create_table(
+        "pandas_docs",
+        data=[
+            {
+                "vector": embeddings.embed_query("Hello World"),
+                "text": "Hello World",
+                "id": "1",
+            }
+        ],
+        mode="overwrite",
+    )
     docsearch = LanceDB.from_documents(documents, embeddings, connection=table)
-    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
+    qa = RetrievalQA.from_chain_type(
+        llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever()
+    )
     return qa.run(query)
 
+
 @stub.function()
 @web_endpoint(method="GET")
 def web(query: str):
@@ -101,6 +110,7 @@ def web(query: str):
         "answer": answer,
     }
 
+
 @stub.function()
 def cli(query: str):
     answer = qanda_langchain(query)

docs/src/examples/nodejs.md

@@ -1,99 +0,0 @@
-# YouTube transcript QA bot with NodeJS
-
-## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
-
-<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
-
-This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how you can use LanceDB's Javascript API to store and manage your data easily.
-
-For this example we're using a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`, to make it easier, we've converted it to a LanceDB `db` already, which you can download and put in a working directory:
-
-```wget -c https://eto-public.s3.us-west-2.amazonaws.com/lancedb_demo.tar.gz -O - | tar -xz -C .```
-
-Now, we'll create a simple app that can:
-1. Take a text based query and search for contexts in our corpus, using embeddings generated from the OpenAI Embedding API.
-2. Create a prompt with the contexts, and call the OpenAI Completion API to answer the text based query.
-
-Dependencies and setup of OpenAI API:
-
-```javascript
-const lancedb = require("vectordb");
-const { Configuration, OpenAIApi } = require("openai");
-
-const configuration = new Configuration({
-    apiKey: process.env.OPENAI_API_KEY,
-    });
-const openai = new OpenAIApi(configuration);
-```
-
-First, let's set our question and the context amount. The context amount will be used to query similar documents in our corpus.
-
-```javascript
-const QUESTION = "who was the 12th person on the moon and when did they land?";
-const CONTEXT_AMOUNT = 3;
-```
-
-Now, let's generate an embedding from this question:
-
-```javascript
-const embeddingResponse = await openai.createEmbedding({
-    model: "text-embedding-ada-002",
-    input: QUESTION,
-});
-
-const embedding = embeddingResponse.data["data"][0]["embedding"];
-```
-
-Once we have the embedding, we can connect to LanceDB (using the database we downloaded earlier), and search through the chatbot table.
-We'll extract 3 similar documents found.
-
-```javascript
-const db = await lancedb.connect('./lancedb');
-const tbl = await db.openTable('chatbot');
-const query = tbl.search(embedding);
-query.limit = CONTEXT_AMOUNT;
-const context = await query.execute();
-```
-
-Let's combine the context together so we can pass it into our prompt:
-
-```javascript
-for (let i = 1; i < context.length; i++) {
-    context[0]["text"] += " " + context[i]["text"];
-}
-```
-
-Lastly, let's construct the prompt. You could play around with this to create more accurate/better prompts to yield results.
-
-```javascript
-const prompt = "Answer the question based on the context below.\n\n" +
-    "Context:\n" +
-    `${context[0]["text"]}\n` +
-    `\n\nQuestion: ${QUESTION}\nAnswer:`;
-```
-
-We pass the prompt, along with the context, to the completion API.
-
-```javascript
-const completion = await openai.createCompletion({
-    model: "text-davinci-003",
-    prompt,
-    temperature: 0,
-    max_tokens: 400,
-    top_p: 1,
-    frequency_penalty: 0,
-    presence_penalty: 0,
-});
-```
-
-And that's it!
-
-```javascript
-console.log(completion.data.choices[0].text);
-```
-
-The response is (which is non deterministic):
-
-```
-The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
-```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -0,0 +1,121 @@
+# Vector embedding search using TransformersJS
+
+## Embed and query data from LanceDB using TransformersJS
+
+<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
+
+This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
+
+
+### Setting up
+First, install the dependencies:
+```bash
+npm install vectordb
+npm i @xenova/transformers
+```
+
+We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
+
+Within our `index.js` file we will import the necessary libraries and define our model and database:
+
+```javascript
+const lancedb = require('vectordb')
+const { pipeline } = await import('@xenova/transformers')
+const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+```
+
+### Creating the embedding function
+
+Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
+
+```javascript
+// Define the function. `sourceColumn` is required for LanceDB to know
+// which column to use as input.
+const embed_fun = {}
+embed_fun.sourceColumn = 'text'
+embed_fun.embed = async function (batch) {
+    let result = []
+    // Given a batch of strings, we will use the `pipe` function to get
+    // the vector embedding of each string.
+    for (let text of batch) {
+        // 'mean' pooling and normalizing allows the embeddings to share the
+        // same length.
+        const res = await pipe(text, { pooling: 'mean', normalize: true })
+        result.push(Array.from(res['data']))
+    }
+    return (result)
+}
+```
+
+### Creating the database
+
+Now, we will create the LanceDB database and add the embedding function we defined earlier.
+
+```javascript
+// Link a folder and create a table with data
+const db = await lancedb.connect('data/sample-lancedb')
+
+// You can also import any other data, but make sure that you have a column
+// for the embedding function to use.
+const data = [
+    { id: 1, text: 'Cherry', type: 'fruit' },
+    { id: 2, text: 'Carrot', type: 'vegetable' },
+    { id: 3, text: 'Potato', type: 'vegetable' },
+    { id: 4, text: 'Apple', type: 'fruit' },
+    { id: 5, text: 'Banana', type: 'fruit' }
+]
+
+// Create the table with the embedding function
+const table = await db.createTable('food_table', data, "create", embed_fun)
+```
+
+### Performing the search
+
+Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
+
+```javascript
+// Query the table
+const results = await table
+    .search("a sweet fruit to eat")
+    .metricType("cosine")
+    .limit(2)
+    .execute()
+console.log(results.map(r => r.text))
+```
+```bash
+[ 'Banana', 'Cherry' ]
+```
+
+Output of `results`:
+```bash
+[
+  {
+    vector: Float32Array(384) [
+      -0.057455405592918396,
+      0.03617725893855095,
+      -0.0367760956287384,
+      ... 381 more items
+    ],
+    id: 5,
+    text: 'Banana',
+    type: 'fruit',
+    score: 0.4919965863227844
+  },
+  {
+    vector: Float32Array(384) [
+      0.0009714411571621895,
+      0.008223623037338257,
+      0.009571489877998829,
+      ... 381 more items
+    ],
+    id: 1,
+    text: 'Cherry',
+    type: 'fruit',
+    score: 0.5540297031402588
+  }
+]
+```
+
+### Wrapping it up
+
+In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/examples/youtube_transcript_bot_with_nodejs.md

@@ -0,0 +1,139 @@
+# YouTube transcript QA bot with NodeJS
+
+## use LanceDB's Javascript API and OpenAI to build a QA bot for YouTube transcripts
+
+<img id="splash" width="400" alt="nodejs" src="https://github.com/lancedb/lancedb/assets/917119/3a140e75-bf8e-438a-a1e4-af14a72bcf98">
+
+This Q&A bot will allow you to search through youtube transcripts using natural language! We'll introduce how to use LanceDB's Javascript API to store and manage your data easily.
+
+```bash
+npm install vectordb
+```
+
+## Download the data
+
+For this example, we're using a sample of a HuggingFace dataset that contains YouTube transcriptions: `jamescalam/youtube-transcriptions`. Download and extract this file under the `data` folder:
+
+```bash
+wget -c https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-transcriptions_sample.jsonl
+```
+
+## Prepare Context
+
+Each item in the dataset contains just a short chunk of text. We'll need to merge a bunch of these chunks together on a rolling basis. For this demo, we'll look back 20 records to create a more complete context for each sentence.
+
+First, we need to read and parse the input file.
+
+```javascript
+const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
+  .toString()
+  .split('\n')
+  .filter(line => line.length > 0)
+  .map(line => JSON.parse(line))
+
+const data = contextualize(lines, 20, 'video_id')
+```
+
+The contextualize function groups the transcripts by video_id and then creates the expanded context for each item.
+
+```javascript
+function contextualize (rows, contextSize, groupColumn) {
+  const grouped = []
+  rows.forEach(row => {
+    if (!grouped[row[groupColumn]]) {
+      grouped[row[groupColumn]] = []
+    }
+    grouped[row[groupColumn]].push(row)
+  })
+
+  const data = []
+  Object.keys(grouped).forEach(key => {
+    for (let i = 0; i < grouped[key].length; i++) {
+      const start = i - contextSize > 0 ? i - contextSize : 0
+      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
+    }
+    data.push(...grouped[key])
+  })
+  return data
+}
+```
+
+## Create the LanceDB Table
+
+To load our data into LanceDB, we need to create embedding (vectors) for each item. For this example, we will use the OpenAI embedding functions, which have a native integration with LanceDB.
+
+```javascript
+// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+const apiKey = process.env.OPENAI_API_KEY
+// The embedding function will create embeddings for the 'context' column
+const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
+// Connects to LanceDB
+const db = await lancedb.connect('data/youtube-lancedb')
+const tbl = await db.createTable('vectors', data, embedFunction)
+```
+
+## Create and answer the prompt
+
+We will accept questions in natural language and use our corpus stored in LanceDB to answer them. First, we need to set up the OpenAI client:
+
+```javascript
+const configuration = new Configuration({ apiKey })
+const openai = new OpenAIApi(configuration)
+```
+
+Then we can prompt questions and use LanceDB to retrieve the three most relevant transcripts for this prompt.
+
+```javascript
+const query = await rl.question('Prompt: ')
+const results = await tbl
+  .search(query)
+  .select(['title', 'text', 'context'])
+  .limit(3)
+  .execute()
+```
+
+The query and the transcripts' context are appended together in a single prompt:
+
+```javascript
+function createPrompt (query, context) {
+    let prompt =
+        'Answer the question based on the context below.\n\n' +
+        'Context:\n'
+
+    // need to make sure our prompt is not larger than max size
+    prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
+    prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
+    return prompt
+}
+```
+
+We can now use the OpenAI Completion API to process our custom prompt and give us an answer.
+
+```javascript
+const response = await openai.createCompletion({
+  model: 'text-davinci-003',
+  prompt: createPrompt(query, results),
+  max_tokens: 400,
+  temperature: 0,
+  top_p: 1,
+  frequency_penalty: 0,
+  presence_penalty: 0
+})
+console.log(response.data.choices[0].text)
+```
+
+## Let's put it all together now
+
+Now we can provide queries and have them answered based on your local LanceDB data.
+
+```bash
+Prompt: who was the 12th person on the moon and when did they land?
+ The 12th person on the moon was Harrison Schmitt and he landed on December 11, 1972.
+Prompt: Which training method should I use for sentence transformers when I only have pairs of related sentences?
+ NLI with multiple negative ranking loss.
+```
+
+## That's a wrap
+
+In this example, you learned how to use LanceDB to store and query embedding representations of your local data. The complete example code is on [GitHub](https://github.com/lancedb/lancedb/tree/main/node/examples), and you can also download the LanceDB dataset using [this link](https://eto-public.s3.us-west-2.amazonaws.com/datasets/youtube_transcript/youtube-lancedb.zip).
+

docs/src/fts.md

@@ -18,6 +18,20 @@ Assume:
 1. `table` is a LanceDB Table
 2. `text` is the name of the Table column that we want to index
 
+For example,
+
+```python
+import lancedb
+
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+table = db.create_table("my_table",
+            data=[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
+                  {"vector": [5.9, 26.5], "text": "There are several kittens playing"}])
+
+```
+
 To create the index:
 
 ```python

docs/src/index.md

@@ -8,11 +8,13 @@ The key features of LanceDB include:
 
 * Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
 
+* Support for vector similarity search, full-text search and SQL.
+
 * Native Python and Javascript/Typescript support.
 
 * Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
 
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lanecdb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
 
 LanceDB's core is written in Rust 🦀 and is built using <a href="https://github.com/lancedb/lance">Lance</a>, an open-source columnar format designed for performant ML workloads.
 
@@ -26,7 +28,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       ```python
       import lancedb
 
-      uri = "/tmp/lancedb"
+      uri = "data/sample-lancedb"
       db = lancedb.connect(uri)
       table = db.create_table("my_table",
                               data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
@@ -42,9 +44,9 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
       ```javascript
       const lancedb = require("vectordb");
 
-      const uri = "/tmp/lancedb";
+      const uri = "data/sample-lancedb";
       const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table", 
+      const table = await db.createTable("my_table",
             [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
             { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
       const results = await table.search([100, 100]).limit(2).execute();
@@ -54,11 +56,17 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 - [YouTube Transcript Search](notebooks/youtube_transcript_search.ipynb)
 - [Documentation QA Bot using LangChain](notebooks/code_qa_bot.ipynb)
 - [Multimodal search using CLIP](notebooks/multimodal_search.ipynb)
+- [Serverless QA Bot with S3 and Lambda](examples/serverless_lancedb_with_s3_and_lambda.md)
+- [Serverless QA Bot with Modal](examples/serverless_qa_bot_with_modal_and_langchain.md)
+
+## Complete Demos (JavaScript)
+- [YouTube Transcript Search](examples/youtube_transcript_bot_with_nodejs.md)
 
 ## Documentation Quick Links
 * [`Basic Operations`](basic.md) - basic functionality of LanceDB.
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
-* [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
-* [`API Reference`](python.md) - detailed documentation for the LanceDB Python SDK.
+* [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
+* [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
+* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations.md

@@ -1,111 +0,0 @@
-# Integrations
-
-Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python ecosystem, including Pandas, PyArrow and DuckDB.
-
-## Pandas and PyArrow
-
-First, we need to connect to a `LanceDB` database.
-
-``` py
-
-import lancedb
-
-db = lancedb.connect("/tmp/lancedb")
-```
-
-And write a `Pandas DataFrame` to LanceDB directly.
-
-```py
-import pandas as pd
-
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-
-# Optionally, create a IVF_PQ index
-table.create_index(num_partitions=256, num_sub_vectors=96)
-```
-
-You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](indexing.md)
-sections.
-
-
-We can now perform similarity searches via `LanceDB`.
-
-```py
-# Open the table previously created.
-table = db.open_table("pd_table")
-
-query_vector = [100, 100]
-# Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
-print(df)
-```
-
-```
-    vector     item  price        score
-0  [5.9, 26.5]  bar   20.0  14257.05957
-```
-
-If you have a simple filter, it's faster to provide a where clause to `LanceDB`'s search query.
-If you have more complex criteria, you can always apply the filter to the resulting pandas `DataFrame` from the search query.
-
-```python
-
-# Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-
-# Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
-results = df[df.price < 15]
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-```
-
-## DuckDB
-
-`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
-
-Let us start with installing `duckdb` and `lancedb`.
-
-```shell
-pip install duckdb lancedb
-```
-
-We will re-use the dataset created previously
-
-```python
-import lancedb
-
-db = lancedb.connect("/tmp/lancedb")
-table = db.open_table("pd_table")
-arrow_table = table.to_arrow()
-```
-
-`DuckDB` can directly query the `arrow_table`:
-
-```python
-In [15]: duckdb.query("SELECT * FROM t")
-Out[15]:
-┌─────────────┬─────────┬────────┐
-│   vector    │  item   │ price  │
-│   float[]   │ varchar │ double │
-├─────────────┼─────────┼────────┤
-│ [3.1, 4.1]  │ foo     │   10.0 │
-│ [5.9, 26.5] │ bar     │   20.0 │
-└─────────────┴─────────┴────────┘
-
-In [16]: duckdb.query("SELECT mean(price) FROM t")
-Out[16]:
-┌─────────────┐
-│ mean(price) │
-│   double    │
-├─────────────┤
-│        15.0 │
-└─────────────┘
-```

docs/src/javascript/README.md

@@ -10,15 +10,21 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
 
 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```
 
@@ -26,17 +32,33 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-The LanceDB javascript is built with npm:
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
+
+```bash
+npm test
+```
+
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript
 
 ```bash
 npm run tsc
 ```
 
-Run the tests with
-
-```bash
-npm test
-```
+### Fix lints
 
 To run the linter and have it automatically fix all errors
 

docs/src/javascript/classes/Connection.md

@@ -1,211 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Connection
-
-# Class: Connection
-
-A connection to a LanceDB database.
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Connection.md#constructor)
-
-### Properties
-
-- [\_db](Connection.md#_db)
-- [\_uri](Connection.md#_uri)
-
-### Accessors
-
-- [uri](Connection.md#uri)
-
-### Methods
-
-- [createTable](Connection.md#createtable)
-- [createTableArrow](Connection.md#createtablearrow)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
-## Constructors
-
-### constructor
-
-• **new Connection**(`db`, `uri`)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `db` | `any` |
-| `uri` | `string` |
-
-#### Defined in
-
-[index.ts:46](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L46)
-
-## Properties
-
-### \_db
-
-• `Private` `Readonly` **\_db**: `any`
-
-#### Defined in
-
-[index.ts:44](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L44)
-
-___
-
-### \_uri
-
-• `Private` `Readonly` **\_uri**: `string`
-
-#### Defined in
-
-[index.ts:43](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L43)
-
-## Accessors
-
-### uri
-
-• `get` **uri**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:51](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L51)
-
-## Methods
-
-### createTable
-
-▸ **createTable**(`name`, `data`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:91](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L91)
-
-▸ **createTable**<`T`\>(`name`, `data`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Creates a new Table and initialize it with new data.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:99](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L99)
-
-___
-
-### createTableArrow
-
-▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `name` | `string` |
-| `table` | `Table`<`any`\> |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:109](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L109)
-
-___
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-Open a table in the database.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`number`[]\>\>
-
-#### Defined in
-
-[index.ts:67](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L67)
-
-▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](Table.md)<`T`\>\>
-
-Open a table in the database.
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
-
-#### Returns
-
-`Promise`<[`Table`](Table.md)<`T`\>\>
-
-#### Defined in
-
-[index.ts:74](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L74)
-
-___
-
-### tableNames
-
-▸ **tableNames**(): `Promise`<`string`[]\>
-
-Get the names of all tables in the database.
-
-#### Returns
-
-`Promise`<`string`[]\>
-
-#### Defined in
-
-[index.ts:58](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L58)

docs/src/javascript/classes/LocalConnection.md

@@ -0,0 +1,350 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalConnection
+
+# Class: LocalConnection
+
+A connection to a LanceDB database.
+
+## Implements
+
+- [`Connection`](../interfaces/Connection.md)
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalConnection.md#constructor)
+
+### Properties
+
+- [\_db](LocalConnection.md#_db)
+- [\_options](LocalConnection.md#_options)
+
+### Accessors
+
+- [uri](LocalConnection.md#uri)
+
+### Methods
+
+- [createTable](LocalConnection.md#createtable)
+- [createTableArrow](LocalConnection.md#createtablearrow)
+- [dropTable](LocalConnection.md#droptable)
+- [openTable](LocalConnection.md#opentable)
+- [tableNames](LocalConnection.md#tablenames)
+
+## Constructors
+
+### constructor
+
+• **new LocalConnection**(`db`, `options`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `db` | `any` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
+
+#### Defined in
+
+[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
+
+## Properties
+
+### \_db
+
+• `Private` `Readonly` **\_db**: `any`
+
+#### Defined in
+
+[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
+
+___
+
+### \_options
+
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Defined in
+
+[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
+
+## Accessors
+
+### uri
+
+• `get` **uri**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[uri](../interfaces/Connection.md#uri)
+
+#### Defined in
+
+[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
+
+## Methods
+
+### createTable
+
+▸ **createTable**(`name`, `data`, `mode?`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTable](../interfaces/Connection.md#createtable)
+
+#### Defined in
+
+[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
+
+▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the Table |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[createTableArrow](../interfaces/Connection.md#createtablearrow)
+
+#### Defined in
+
+[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[dropTable](../interfaces/Connection.md#droptable)
+
+#### Defined in
+
+[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
+
+___
+
+### openTable
+
+▸ **openTable**(`name`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+Open a table in the database.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[openTable](../interfaces/Connection.md#opentable)
+
+#### Defined in
+
+[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
+
+▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use on this Table |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+Get the names of all tables in the database.
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Implementation of
+
+[Connection](../interfaces/Connection.md).[tableNames](../interfaces/Connection.md#tablenames)
+
+#### Defined in
+
+[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)

docs/src/javascript/classes/LocalTable.md

@@ -0,0 +1,302 @@
+[vectordb](../README.md) / [Exports](../modules.md) / LocalTable
+
+# Class: LocalTable<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implements
+
+- [`Table`](../interfaces/Table.md)<`T`\>
+
+## Table of contents
+
+### Constructors
+
+- [constructor](LocalTable.md#constructor)
+
+### Properties
+
+- [\_embeddings](LocalTable.md#_embeddings)
+- [\_name](LocalTable.md#_name)
+- [\_options](LocalTable.md#_options)
+- [\_tbl](LocalTable.md#_tbl)
+
+### Accessors
+
+- [name](LocalTable.md#name)
+
+### Methods
+
+- [add](LocalTable.md#add)
+- [countRows](LocalTable.md#countrows)
+- [createIndex](LocalTable.md#createindex)
+- [delete](LocalTable.md#delete)
+- [overwrite](LocalTable.md#overwrite)
+- [search](LocalTable.md#search)
+
+## Constructors
+
+### constructor
+
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `tbl` | `any` |
+| `name` | `string` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
+
+#### Defined in
+
+[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
+
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
+
+#### Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `tbl` | `any` |  |
+| `name` | `string` |  |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
+| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
+
+#### Defined in
+
+[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
+
+## Properties
+
+### \_embeddings
+
+• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
+
+#### Defined in
+
+[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
+
+___
+
+### \_name
+
+• `Private` `Readonly` **\_name**: `string`
+
+#### Defined in
+
+[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
+
+___
+
+### \_options
+
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Defined in
+
+[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
+
+___
+
+### \_tbl
+
+• `Private` `Readonly` **\_tbl**: `any`
+
+#### Defined in
+
+[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
+
+## Accessors
+
+### name
+
+• `get` **name**(): `string`
+
+#### Returns
+
+`string`
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[name](../interfaces/Table.md#name)
+
+#### Defined in
+
+[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
+
+## Methods
+
+### add
+
+▸ **add**(`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[add](../interfaces/Table.md#add)
+
+#### Defined in
+
+[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
+
+___
+
+### countRows
+
+▸ **countRows**(): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+#### Returns
+
+`Promise`<`number`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[countRows](../interfaces/Table.md#countrows)
+
+#### Defined in
+
+[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
+
+___
+
+### createIndex
+
+▸ **createIndex**(`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[createIndex](../interfaces/Table.md#createindex)
+
+#### Defined in
+
+[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
+
+___
+
+### delete
+
+▸ **delete**(`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[delete](../interfaces/Table.md#delete)
+
+#### Defined in
+
+[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
+
+___
+
+### overwrite
+
+▸ **overwrite**(`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+#### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[overwrite](../interfaces/Table.md#overwrite)
+
+#### Defined in
+
+[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
+
+___
+
+### search
+
+▸ **search**(`query`): [`Query`](Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Implementation of
+
+[Table](../interfaces/Table.md).[search](../interfaces/Table.md#search)
+
+#### Defined in
+
+[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -18,7 +18,6 @@ A builder for nearest neighbor queries for LanceDB.
 
 ### Properties
 
-- [\_columns](Query.md#_columns)
 - [\_embeddings](Query.md#_embeddings)
 - [\_filter](Query.md#_filter)
 - [\_limit](Query.md#_limit)
@@ -27,7 +26,9 @@ A builder for nearest neighbor queries for LanceDB.
 - [\_query](Query.md#_query)
 - [\_queryVector](Query.md#_queryvector)
 - [\_refineFactor](Query.md#_refinefactor)
+- [\_select](Query.md#_select)
 - [\_tbl](Query.md#_tbl)
+- [where](Query.md#where)
 
 ### Methods
 
@@ -37,6 +38,7 @@ A builder for nearest neighbor queries for LanceDB.
 - [metricType](Query.md#metrictype)
 - [nprobes](Query.md#nprobes)
 - [refineFactor](Query.md#refinefactor)
+- [select](Query.md#select)
 
 ## Constructors
 
@@ -60,27 +62,17 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:241](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L241)
+[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
 
 ## Properties
 
-### \_columns
-
-• `Private` `Optional` `Readonly` **\_columns**: `string`[]
-
-#### Defined in
-
-[index.ts:236](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L236)
-
-___
-
 ### \_embeddings
 
 • `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
 
 #### Defined in
 
-[index.ts:239](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L239)
+[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
 
 ___
 
@@ -90,7 +82,7 @@ ___
 
 #### Defined in
 
-[index.ts:237](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L237)
+[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
 
 ___
 
@@ -100,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:233](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L233)
+[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
 
 ___
 
@@ -110,7 +102,7 @@ ___
 
 #### Defined in
 
-[index.ts:238](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L238)
+[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
 
 ___
 
@@ -120,7 +112,7 @@ ___
 
 #### Defined in
 
-[index.ts:235](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L235)
+[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
 
 ___
 
@@ -130,7 +122,7 @@ ___
 
 #### Defined in
 
-[index.ts:231](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L231)
+[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
 
 ___
 
@@ -140,7 +132,7 @@ ___
 
 #### Defined in
 
-[index.ts:232](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L232)
+[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
 
 ___
 
@@ -150,7 +142,17 @@ ___
 
 #### Defined in
 
-[index.ts:234](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L234)
+[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
+
+___
+
+### \_select
+
+• `Private` `Optional` **\_select**: `string`[]
+
+#### Defined in
+
+[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
 
 ___
 
@@ -160,7 +162,33 @@ ___
 
 #### Defined in
 
-[index.ts:230](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L230)
+[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
+
+___
+
+### where
+
+• **where**: (`value`: `string`) => [`Query`](Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`value`): [`Query`](Query.md)<`T`\>
+
+A filter statement to be applied to this query.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string` | A filter in the same format used by a sql WHERE clause. |
+
+##### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
 
 ## Methods
 
@@ -182,7 +210,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[index.ts:301](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L301)
+[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
 
 ___
 
@@ -204,7 +232,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:284](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L284)
+[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
 
 ___
 
@@ -226,7 +254,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[index.ts:257](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L257)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
 
 ___
 
@@ -252,7 +280,7 @@ MetricType for the different options
 
 #### Defined in
 
-[index.ts:293](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L293)
+[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
 
 ___
 
@@ -274,7 +302,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[index.ts:275](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L275)
+[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
 
 ___
 
@@ -296,4 +324,26 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[index.ts:266](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L266)
+[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
+
+___
+
+### select
+
+▸ **select**(`value`): [`Query`](Query.md)<`T`\>
+
+Return only the specified columns.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `value` | `string`[] | Only select the specified columns. If not specified, all columns will be returned. |
+
+#### Returns
+
+[`Query`](Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)

docs/src/javascript/classes/Table.md

@@ -1,215 +0,0 @@
-[vectordb](../README.md) / [Exports](../modules.md) / Table
-
-# Class: Table<T\>
-
-## Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Table.md#constructor)
-
-### Properties
-
-- [\_embeddings](Table.md#_embeddings)
-- [\_name](Table.md#_name)
-- [\_tbl](Table.md#_tbl)
-
-### Accessors
-
-- [name](Table.md#name)
-
-### Methods
-
-- [add](Table.md#add)
-- [create\_index](Table.md#create_index)
-- [overwrite](Table.md#overwrite)
-- [search](Table.md#search)
-
-## Constructors
-
-### constructor
-
-• **new Table**<`T`\>(`tbl`, `name`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `tbl` | `any` |
-| `name` | `string` |
-
-#### Defined in
-
-[index.ts:121](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L121)
-
-• **new Table**<`T`\>(`tbl`, `name`, `embeddings`)
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `T` | `number`[] |
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `tbl` | `any` |  |
-| `name` | `string` |  |
-| `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
-
-#### Defined in
-
-[index.ts:127](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L127)
-
-## Properties
-
-### \_embeddings
-
-• `Private` `Optional` `Readonly` **\_embeddings**: [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\>
-
-#### Defined in
-
-[index.ts:119](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L119)
-
-___
-
-### \_name
-
-• `Private` `Readonly` **\_name**: `string`
-
-#### Defined in
-
-[index.ts:118](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L118)
-
-___
-
-### \_tbl
-
-• `Private` `Readonly` **\_tbl**: `any`
-
-#### Defined in
-
-[index.ts:117](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L117)
-
-## Accessors
-
-### name
-
-• `get` **name**(): `string`
-
-#### Returns
-
-`string`
-
-#### Defined in
-
-[index.ts:134](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L134)
-
-## Methods
-
-### add
-
-▸ **add**(`data`): `Promise`<`number`\>
-
-Insert records into this Table.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:152](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L152)
-
-___
-
-### create\_index
-
-▸ **create_index**(`indexParams`): `Promise`<`any`\>
-
-Create an ANN index on this Table vector index.
-
-**`See`**
-
-VectorIndexParams.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
-
-#### Returns
-
-`Promise`<`any`\>
-
-#### Defined in
-
-[index.ts:171](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L171)
-
-___
-
-### overwrite
-
-▸ **overwrite**(`data`): `Promise`<`number`\>
-
-Insert records into this Table, replacing its contents.
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
-
-#### Returns
-
-`Promise`<`number`\>
-
-The number of rows added to the table
-
-#### Defined in
-
-[index.ts:162](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L162)
-
-___
-
-### search
-
-▸ **search**(`query`): [`Query`](Query.md)<`T`\>
-
-Creates a search query to find the nearest neighbors of the given search term
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `query` | `T` | The query search term |
-
-#### Returns
-
-[`Query`](Query.md)<`T`\>
-
-#### Defined in
-
-[index.ts:142](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L142)

docs/src/javascript/enums/MetricType.md

@@ -9,6 +9,7 @@ Distance metrics type.
 ### Enumeration Members
 
 - [Cosine](MetricType.md#cosine)
+- [Dot](MetricType.md#dot)
 - [L2](MetricType.md#l2)
 
 ## Enumeration Members
@@ -21,7 +22,19 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:341](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L341)
+[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
+
+___
+
+### Dot
+
+• **Dot** = ``"dot"``
+
+Dot product
+
+#### Defined in
+
+[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
 
 ___
 
@@ -33,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:336](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L336)
+[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)

docs/src/javascript/enums/WriteMode.md

@@ -2,11 +2,14 @@
 
 # Enumeration: WriteMode
 
+Write mode for writing a table.
+
 ## Table of contents
 
 ### Enumeration Members
 
 - [Append](WriteMode.md#append)
+- [Create](WriteMode.md#create)
 - [Overwrite](WriteMode.md#overwrite)
 
 ## Enumeration Members
@@ -15,9 +18,23 @@
 
 • **Append** = ``"append"``
 
+Append new data to the table.
+
 #### Defined in
 
-[index.ts:326](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L326)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
+
+___
+
+### Create
+
+• **Create** = ``"create"``
+
+Create a new [Table](../interfaces/Table.md).
+
+#### Defined in
+
+[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
 
 ___
 
@@ -25,6 +42,8 @@ ___
 
 • **Overwrite** = ``"overwrite"``
 
+Overwrite the existing [Table](../interfaces/Table.md) if presented.
+
 #### Defined in
 
-[index.ts:325](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L325)
+[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / AwsCredentials
+
+# Interface: AwsCredentials
+
+## Table of contents
+
+### Properties
+
+- [accessKeyId](AwsCredentials.md#accesskeyid)
+- [secretKey](AwsCredentials.md#secretkey)
+- [sessionToken](AwsCredentials.md#sessiontoken)
+
+## Properties
+
+### accessKeyId
+
+• **accessKeyId**: `string`
+
+#### Defined in
+
+[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
+
+___
+
+### secretKey
+
+• **secretKey**: `string`
+
+#### Defined in
+
+[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
+
+___
+
+### sessionToken
+
+• `Optional` **sessionToken**: `string`
+
+#### Defined in
+
+[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/Connection.md

@@ -0,0 +1,152 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Connection
+
+# Interface: Connection
+
+A LanceDB Connection that allows you to open tables and create new ones.
+
+Connection could be local against filesystem or remote against a server.
+
+## Implemented by
+
+- [`LocalConnection`](../classes/LocalConnection.md)
+
+## Table of contents
+
+### Properties
+
+- [uri](Connection.md#uri)
+
+### Methods
+
+- [createTable](Connection.md#createtable)
+- [createTableArrow](Connection.md#createtablearrow)
+- [dropTable](Connection.md#droptable)
+- [openTable](Connection.md#opentable)
+- [tableNames](Connection.md#tablenames)
+
+## Properties
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
+
+## Methods
+
+### createTable
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode?`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Creates a new Table and initialize it with new data.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `data` | `Record`<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
+| `mode?` | [`WriteMode`](../enums/WriteMode.md) | The write mode to use when creating the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
+
+___
+
+### createTableArrow
+
+▸ **createTableArrow**(`name`, `table`): `Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `table` | `Table`<`any`\> |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`number`[]\>\>
+
+#### Defined in
+
+[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
+
+___
+
+### dropTable
+
+▸ **dropTable**(`name`): `Promise`<`void`\>
+
+Drop an existing table.
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table to drop. |
+
+#### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
+
+___
+
+### openTable
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](Table.md)<`T`\>\>
+
+Open a table in the database.
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `name` | `string` | The name of the table. |
+| `embeddings?` | [`EmbeddingFunction`](EmbeddingFunction.md)<`T`\> | An embedding function to use on this table |
+
+#### Returns
+
+`Promise`<[`Table`](Table.md)<`T`\>\>
+
+#### Defined in
+
+[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
+
+___
+
+### tableNames
+
+▸ **tableNames**(): `Promise`<`string`[]\>
+
+#### Returns
+
+`Promise`<`string`[]\>
+
+#### Defined in
+
+[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -0,0 +1,30 @@
+[vectordb](../README.md) / [Exports](../modules.md) / ConnectionOptions
+
+# Interface: ConnectionOptions
+
+## Table of contents
+
+### Properties
+
+- [awsCredentials](ConnectionOptions.md#awscredentials)
+- [uri](ConnectionOptions.md#uri)
+
+## Properties
+
+### awsCredentials
+
+• `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
+
+#### Defined in
+
+[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
+
+___
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/31dab97/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -0,0 +1,149 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IvfPQIndexConfig
+
+# Interface: IvfPQIndexConfig
+
+## Table of contents
+
+### Properties
+
+- [column](IvfPQIndexConfig.md#column)
+- [index\_name](IvfPQIndexConfig.md#index_name)
+- [max\_iters](IvfPQIndexConfig.md#max_iters)
+- [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
+- [metric\_type](IvfPQIndexConfig.md#metric_type)
+- [num\_bits](IvfPQIndexConfig.md#num_bits)
+- [num\_partitions](IvfPQIndexConfig.md#num_partitions)
+- [num\_sub\_vectors](IvfPQIndexConfig.md#num_sub_vectors)
+- [replace](IvfPQIndexConfig.md#replace)
+- [type](IvfPQIndexConfig.md#type)
+- [use\_opq](IvfPQIndexConfig.md#use_opq)
+
+## Properties
+
+### column
+
+• `Optional` **column**: `string`
+
+The column to be indexed
+
+#### Defined in
+
+[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
+
+___
+
+### index\_name
+
+• `Optional` **index\_name**: `string`
+
+A unique name for the index
+
+#### Defined in
+
+[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
+
+___
+
+### max\_iters
+
+• `Optional` **max\_iters**: `number`
+
+The max number of iterations for kmeans training.
+
+#### Defined in
+
+[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
+
+___
+
+### max\_opq\_iters
+
+• `Optional` **max\_opq\_iters**: `number`
+
+Max number of iterations to train OPQ, if `use_opq` is true.
+
+#### Defined in
+
+[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
+
+___
+
+### metric\_type
+
+• `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
+
+Metric type, L2 or Cosine
+
+#### Defined in
+
+[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
+
+___
+
+### num\_bits
+
+• `Optional` **num\_bits**: `number`
+
+The number of bits to present one PQ centroid.
+
+#### Defined in
+
+[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
+
+___
+
+### num\_partitions
+
+• `Optional` **num\_partitions**: `number`
+
+The number of partitions this index
+
+#### Defined in
+
+[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
+
+___
+
+### num\_sub\_vectors
+
+• `Optional` **num\_sub\_vectors**: `number`
+
+Number of subvectors to build PQ code
+
+#### Defined in
+
+[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
+
+___
+
+### replace
+
+• `Optional` **replace**: `boolean`
+
+Replace an existing index with the same name if it exists.
+
+#### Defined in
+
+[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
+
+___
+
+### type
+
+• **type**: ``"ivf_pq"``
+
+#### Defined in
+
+[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
+
+___
+
+### use\_opq
+
+• `Optional` **use\_opq**: `boolean`
+
+Train as optimized product quantization.
+
+#### Defined in
+
+[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -0,0 +1,221 @@
+[vectordb](../README.md) / [Exports](../modules.md) / Table
+
+# Interface: Table<T\>
+
+A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+
+## Type parameters
+
+| Name | Type |
+| :------ | :------ |
+| `T` | `number`[] |
+
+## Implemented by
+
+- [`LocalTable`](../classes/LocalTable.md)
+
+## Table of contents
+
+### Properties
+
+- [add](Table.md#add)
+- [countRows](Table.md#countrows)
+- [createIndex](Table.md#createindex)
+- [delete](Table.md#delete)
+- [name](Table.md#name)
+- [overwrite](Table.md#overwrite)
+- [search](Table.md#search)
+
+## Properties
+
+### add
+
+• **add**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
+
+___
+
+### countRows
+
+• **countRows**: () => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (): `Promise`<`number`\>
+
+Returns the number of rows in this table.
+
+##### Returns
+
+`Promise`<`number`\>
+
+#### Defined in
+
+[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
+
+___
+
+### createIndex
+
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
+
+#### Type declaration
+
+▸ (`indexParams`): `Promise`<`any`\>
+
+Create an ANN index on this Table vector index.
+
+**`See`**
+
+VectorIndexParams.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `indexParams` | [`IvfPQIndexConfig`](IvfPQIndexConfig.md) | The parameters of this Index, |
+
+##### Returns
+
+`Promise`<`any`\>
+
+#### Defined in
+
+[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
+
+___
+
+### delete
+
+• **delete**: (`filter`: `string`) => `Promise`<`void`\>
+
+#### Type declaration
+
+▸ (`filter`): `Promise`<`void`\>
+
+Delete rows from this table.
+
+This can be used to delete a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [1, 2]},
+   {id: 2, vector: [3, 4]},
+   {id: 3, vector: [5, 6]},
+];
+const tbl = await con.createTable("my_table", data)
+await tbl.delete("id = 2")
+await tbl.countRows() // Returns 2
+```
+
+If you have a list of values to delete, you can combine them into a
+stringified list and use the `IN` operator:
+
+```ts
+const to_remove = [1, 5];
+await tbl.delete(`id IN (${to_remove.join(",")})`)
+await tbl.countRows() // Returns 1
+```
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
+
+##### Returns
+
+`Promise`<`void`\>
+
+#### Defined in
+
+[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
+
+___
+
+### name
+
+• **name**: `string`
+
+#### Defined in
+
+[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
+
+___
+
+### overwrite
+
+• **overwrite**: (`data`: `Record`<`string`, `unknown`\>[]) => `Promise`<`number`\>
+
+#### Type declaration
+
+▸ (`data`): `Promise`<`number`\>
+
+Insert records into this Table, replacing its contents.
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `data` | `Record`<`string`, `unknown`\>[] | Records to be inserted into the Table |
+
+##### Returns
+
+`Promise`<`number`\>
+
+The number of rows added to the table
+
+#### Defined in
+
+[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
+
+___
+
+### search
+
+• **search**: (`query`: `T`) => [`Query`](../classes/Query.md)<`T`\>
+
+#### Type declaration
+
+▸ (`query`): [`Query`](../classes/Query.md)<`T`\>
+
+Creates a search query to find the nearest neighbors of the given search term
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `query` | `T` | The query search term |
+
+##### Returns
+
+[`Query`](../classes/Query.md)<`T`\>
+
+#### Defined in
+
+[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)

docs/src/javascript/modules.md

@@ -11,14 +11,19 @@
 
 ### Classes
 
-- [Connection](classes/Connection.md)
+- [LocalConnection](classes/LocalConnection.md)
+- [LocalTable](classes/LocalTable.md)
 - [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
 - [Query](classes/Query.md)
-- [Table](classes/Table.md)
 
 ### Interfaces
 
+- [AwsCredentials](interfaces/AwsCredentials.md)
+- [Connection](interfaces/Connection.md)
+- [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
+- [Table](interfaces/Table.md)
 
 ### Type Aliases
 
@@ -32,17 +37,17 @@
 
 ### VectorIndexParams
 
-Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
+Ƭ **VectorIndexParams**: [`IvfPQIndexConfig`](interfaces/IvfPQIndexConfig.md)
 
 #### Defined in
 
-[index.ts:224](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L224)
+[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
 
 ## Functions
 
 ### connect
 
-▸ **connect**(`uri`): `Promise`<[`Connection`](classes/Connection.md)\>
+▸ **connect**(`uri`): `Promise`<[`Connection`](interfaces/Connection.md)\>
 
 Connect to a LanceDB instance at the given URI
 
@@ -54,8 +59,24 @@ Connect to a LanceDB instance at the given URI
 
 #### Returns
 
-`Promise`<[`Connection`](classes/Connection.md)\>
+`Promise`<[`Connection`](interfaces/Connection.md)\>
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/31dab97/node/src/index.ts#L34)
+[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
+
+▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+
+#### Returns
+
+`Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Defined in
+
+[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)

docs/src/notebooks/code_qa_bot.ipynb

@@ -181,7 +181,7 @@
    "id": "c3852dd3",
    "metadata": {},
    "source": [
-    "# Generating emebeddings from our docs\n",
+    "# Generating embeddings from our docs\n",
     "\n",
     "Now that we have our raw documents loaded, we need to pre-process them to generate embeddings:"
    ]

docs/src/notebooks/diffusiondb/datagen.py

@@ -21,12 +21,13 @@ from argparse import ArgumentParser
 from multiprocessing import Pool
 
 import lance
-import lancedb
 import pyarrow as pa
 from datasets import load_dataset
 from PIL import Image
 from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
 
+import lancedb
+
 MODEL_ID = "openai/clip-vit-base-patch32"
 
 device = "cuda"

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -1,11 +1,12 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "42bf01fb",
    "metadata": {},
    "source": [
-    "# We're going to build question and answer bot\n",
+    "# Youtube Transcript Search QA Bot\n",
     "\n",
     "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily."
    ]
@@ -35,6 +36,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "22e570f4",
    "metadata": {},
@@ -87,6 +89,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "5ac2b6a3",
    "metadata": {},
@@ -181,6 +184,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "3044e0b0",
    "metadata": {},
@@ -209,6 +213,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "db586267",
    "metadata": {},
@@ -229,6 +234,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "2106b5bb",
    "metadata": {},
@@ -338,6 +344,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "53e4bff1",
    "metadata": {},
@@ -371,6 +378,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "8ef34fca",
    "metadata": {},
@@ -459,6 +467,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "23afc2f9",
    "metadata": {},
@@ -541,6 +550,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "28705959",
    "metadata": {},
@@ -571,6 +581,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "id": "559a095b",
    "metadata": {},

docs/src/python/arrow.md

@@ -0,0 +1,101 @@
+# Pandas and PyArrow
+
+
+Built on top of [Apache Arrow](https://arrow.apache.org/),
+`LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
+and PyArrow.
+
+## Create dataset
+
+First, we need to connect to a `LanceDB` database.
+
+```py
+
+import lancedb
+
+db = lancedb.connect("data/sample-lancedb")
+```
+
+Afterwards, we write a `Pandas DataFrame` to LanceDB directly.
+
+```py
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+```
+
+Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
+[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
+
+For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
+to lazily generate data:
+
+```py
+
+from typing import Iterable
+import pyarrow as pa
+import lancedb
+
+def make_batches() -> Iterable[pa.RecordBatch]:
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"])
+
+schema=pa.schema([
+    pa.field("vector", pa.list_(pa.float32())),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+table = db.create_table("iterable_table", data=make_batches(), schema=schema)
+```
+
+You will find detailed instructions of creating dataset in
+[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
+sections.
+
+## Vector Search
+
+We can now perform similarity search via `LanceDB` Python API.
+
+```py
+# Open the table previously created.
+table = db.open_table("pd_table")
+
+query_vector = [100, 100]
+# Pandas DataFrame
+df = table.search(query_vector).limit(1).to_df()
+print(df)
+```
+
+```
+    vector     item  price        score
+0  [5.9, 26.5]  bar   20.0  14257.05957
+```
+
+If you have a simple filter, it's faster to provide a `where clause` to `LanceDB`'s search query.
+If you have more complex criteria, you can always apply the filter to the resulting Pandas `DataFrame`.
+
+```python
+
+# Apply the filter via LanceDB
+results = table.search([100, 100]).where("price < 15").to_df()
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+
+# Apply the filter via Pandas
+df = results = table.search([100, 100]).to_df()
+results = df[df.price < 15]
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+```

docs/src/python/duckdb.md

@@ -0,0 +1,56 @@
+# DuckDB
+
+`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
+
+Let us start with installing `duckdb` and `lancedb`.
+
+```shell
+pip install duckdb lancedb
+```
+
+We will re-use [the dataset created previously](./arrow.md):
+
+```python
+import pandas as pd
+import lancedb
+
+db = lancedb.connect("data/sample-lancedb")
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+arrow_table = table.to_arrow()
+```
+
+`DuckDB` can directly query the `arrow_table`:
+
+```python
+import duckdb
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+```
+
+```py
+duckdb.query("SELECT mean(price) FROM arrow_table")
+```
+
+```
+┌─────────────┐
+│ mean(price) │
+│   double    │
+├─────────────┤
+│        15.0 │
+└─────────────┘
+```

docs/src/python/integration.md

@@ -0,0 +1,7 @@
+# Integration
+
+Built on top of [Apache Arrow](https://arrow.apache.org/),
+`LanceDB` is very easy to be integrate with Python ecosystems.
+
+* [Pandas and Arrow Integration](./arrow.md)
+* [DuckDB Integration](./duckdb.md)

docs/src/python/pydantic.md

@@ -0,0 +1,35 @@
+# Pydantic
+
+[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
+
+## Schema
+
+LanceDB supports to create Apache Arrow Schema from a
+[Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
+via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
+
+::: lancedb.pydantic.pydantic_to_schema
+
+## Vector Field
+
+LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
+vector Field in a Pydantic Model.
+
+::: lancedb.pydantic.vector
+
+## Type Conversion
+
+LanceDB automatically convert Pydantic fields to
+[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
+
+Current supported type conversions:
+
+| Pydantic Field Type | PyArrow Data Type |
+| ------------------- | ----------------- |
+| `int`               | `pyarrow.int64`   |
+| `float`              | `pyarrow.float64`  |
+| `bool`              | `pyarrow.bool`    |
+| `str`               | `pyarrow.utf8()`    |
+| `list`              | `pyarrow.List`    |
+| `BaseModel`         | `pyarrow.Struct`    |
+| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -6,9 +6,54 @@
 pip install lancedb
 ```
 
-## ::: lancedb
-## ::: lancedb.db
-## ::: lancedb.table
-## ::: lancedb.query
-## ::: lancedb.embeddings
-## ::: lancedb.context
+## Connection
+
+::: lancedb.connect
+
+::: lancedb.db.DBConnection
+
+## Table
+
+::: lancedb.table.Table
+
+## Querying
+
+::: lancedb.query.Query
+
+::: lancedb.query.LanceQueryBuilder
+
+::: lancedb.query.LanceFtsQueryBuilder
+
+## Embeddings
+
+::: lancedb.embeddings.with_embeddings
+
+::: lancedb.embeddings.EmbeddingFunction
+
+## Context
+
+::: lancedb.context.contextualize
+
+::: lancedb.context.Contextualizer
+
+## Full text search
+
+::: lancedb.fts.create_index
+
+::: lancedb.fts.populate_index
+
+::: lancedb.fts.search_index
+
+## Utilities
+
+::: lancedb.vector
+
+## Integrations
+
+### Pydantic
+
+::: lancedb.pydantic.pydantic_to_schema
+
+::: lancedb.pydantic.vector
+
+

docs/src/search.md

@@ -18,26 +18,55 @@ Currently, we support the following metrics:
 | ----------- | ------------------------------------ |
 | `L2`        | [Euclidean / L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) |
 | `Cosine`    | [Cosine Similarity](https://en.wikipedia.org/wiki/Cosine_similarity)|
+| `Dot`       | [Dot Production](https://en.wikipedia.org/wiki/Dot_product) |
 
 
 ## Search
 
 ### Flat Search
 
+If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
+and compute the distance for each vector in order to find the closest matches.
 
-If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
-the vector column and compute the distance.
 
+<!-- Setup Code
+```python
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 1536)).astype('float32'))]
+
+db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript
+const vectordb_setup = require('vectordb')
+const db_setup = await vectordb_setup.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+await db_setup.createTable('my_vectors', data)
+```
+-->
 === "Python"
 
+
     ```python
     import lancedb
+    import numpy as np
+
     db = lancedb.connect("data/sample-lancedb")
 
     tbl = db.open_table("my_vectors")
 
-    df = tbl.search(np.random.random((768)))
-        .limit(10)
+    df = tbl.search(np.random.random((1536))) \
+        .limit(10) \
         .to_df()
     ```
 
@@ -47,10 +76,10 @@ the vector column and compute the distance.
     const vectordb = require('vectordb')
     const db = await vectordb.connect('data/sample-lancedb')
 
-    tbl = db.open_table("my_vectors")
+    const tbl = await db.openTable("my_vectors")
 
-    const results = await tbl.search(Array(768))
-        .limit(20)
+    const results_1 = await tbl.search(Array(1536).fill(1.2))
+        .limit(10)
         .execute()
     ```
 
@@ -60,26 +89,33 @@ as well.
 === "Python"
 
     ```python
-    df = tbl.search(np.random.random((768)))
-        .metric("cosine")
-        .limit(10)
+    df = tbl.search(np.random.random((1536))) \
+        .metric("cosine") \
+        .limit(10) \
         .to_df()
     ```
 
+
 === "JavaScript"
 
     ```javascript
-    const vectordb = require('vectordb')
-    const db = await vectordb.connect('data/sample-lancedb')
-
-    tbl = db.open_table("my_vectors")
-
-    const results = await tbl.search(Array(768))
-        .metric("cosine")
-        .limit(20)
+    const results_2 = await tbl.search(Array(1536).fill(1.2))
+        .metricType("cosine")
+        .limit(10)
         .execute()
     ```
 
-### Search with Vector Index.
+
+### Approximate Nearest Neighbor (ANN) Search with Vector Index.
+
+To accelerate vector retrievals, it is common to build vector indices.
+A vector index is a data structure specifically designed to efficiently organize and
+search vector data based on their similarity or distance metrics.
+By constructing a vector index, you can reduce the search space and avoid the need
+for brute-force scanning of the entire vector column.
+
+However, fast vector search using indices often entails making a trade-off with accuracy to some extent.
+This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
+always returns 100% recall.
 
 See [ANN Index](ann_indexes.md) for more details.

docs/src/sql.md

@@ -0,0 +1,120 @@
+# SQL filters
+
+LanceDB embraces the utilization of standard SQL expressions as predicates for hybrid
+filters. It can be used during hybrid vector search and deletion operations.
+
+Currently, Lance supports a growing list of expressions.
+
+* ``>``, ``>=``, ``<``, ``<=``, ``=``
+* ``AND``, ``OR``, ``NOT``
+* ``IS NULL``, ``IS NOT NULL``
+* ``IS TRUE``, ``IS NOT TRUE``, ``IS FALSE``, ``IS NOT FALSE``
+* ``IN``
+* ``LIKE``, ``NOT LIKE``
+* ``CAST``
+* ``regexp_match(column, pattern)``
+
+For example, the following filter string is acceptable:
+<!-- Setup Code
+```python 
+import lancedb
+import numpy as np
+uri = "data/sample-lancedb"
+db = lancedb.connect(uri)
+
+data = [{"vector": row, "item": f"item {i}"}
+     for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
+
+tbl = db.create_table("my_vectors", data=data)
+```
+-->
+<!-- Setup Code
+```javascript 
+const vectordb = require('vectordb')
+const db = await vectordb.connect('data/sample-lancedb')
+
+let data = []
+for (let i = 0; i < 10_000; i++) {
+     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+}
+const tbl = await db.createTable('my_vectors', data)
+```
+-->
+=== "Python"
+
+    ```python
+    tbl.search([100, 102]) \
+       .where("""(
+        (label IN [10, 20])
+        AND
+        (note.email IS NOT NULL)
+        ) OR NOT note.created
+        """)
+
+    ```
+=== "Javascript"
+
+    ```javascript
+    tbl.search([100, 102])
+       .where(`(
+            (label IN [10, 20])
+            AND
+            (note.email IS NOT NULL)
+        ) OR NOT note.created
+       `)
+    ```
+
+
+If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
+you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
+path must be wrapped in backticks.
+
+=== "SQL"
+    ```sql
+    `CUBE` = 10 AND `column name with space` IS NOT NULL
+      AND `nested with space`.`inner with space` < 2
+    ```
+
+!!! warning
+    Field names containing periods (``.``) are not supported.
+
+Literals for dates, timestamps, and decimals can be written by writing the string
+value after the type name. For example
+
+=== "SQL"
+    ```sql
+    date_col = date '2021-01-01'
+    and timestamp_col = timestamp '2021-01-01 00:00:00'
+    and decimal_col = decimal(8,3) '1.000'
+    ```
+
+For timestamp columns, the precision can be specified as a number in the type
+parameter. Microsecond precision (6) is the default.
+
+| SQL              | Time unit    |
+|------------------|--------------|
+| ``timestamp(0)`` | Seconds      |
+| ``timestamp(3)`` | Milliseconds |
+| ``timestamp(6)`` | Microseconds |
+| ``timestamp(9)`` | Nanoseconds  |
+
+LanceDB internally stores data in [Apache Arrow](https://arrow.apache.org/) format.
+The mapping from SQL types to Arrow types is:
+
+| SQL type | Arrow type |
+|----------|------------|
+| ``boolean`` | ``Boolean`` |
+| ``tinyint`` / ``tinyint unsigned`` | ``Int8`` / ``UInt8`` |
+| ``smallint`` / ``smallint unsigned`` | ``Int16`` / ``UInt16`` |
+| ``int`` or ``integer`` / ``int unsigned`` or ``integer unsigned`` | ``Int32`` / ``UInt32`` |
+| ``bigint`` / ``bigint unsigned`` | ``Int64`` / ``UInt64`` |
+| ``float`` | ``Float32`` |
+| ``double`` | ``Float64`` |
+| ``decimal(precision, scale)`` | ``Decimal128`` |
+| ``date`` | ``Date32`` |
+| ``timestamp`` | ``Timestamp`` [^1] |
+| ``string`` | ``Utf8`` |
+| ``binary`` | ``Binary`` |
+
+[^1]: See precision mapping in previous table.
+

docs/test/md_testing.js

@@ -0,0 +1,52 @@
+const glob = require("glob");
+const fs = require("fs");
+const path = require("path");
+
+const excludedFiles = [
+  "../src/fts.md",
+  "../src/embedding.md",
+  "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+  "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/transformerjs_embedding_search_nodejs.md",
+  "../src/examples/youtube_transcript_bot_with_nodejs.md",
+];
+const nodePrefix = "javascript";
+const nodeFile = ".js";
+const nodeFolder = "node";
+const globString = "../src/**/*.md";
+const asyncPrefix = "(async () => {\n";
+const asyncSuffix = "})();";
+
+function* yieldLines(lines, prefix, suffix) {
+  let inCodeBlock = false;
+  for (const line of lines) {
+    if (line.trim().startsWith(prefix + nodePrefix)) {
+      inCodeBlock = true;
+    } else if (inCodeBlock && line.trim().startsWith(suffix)) {
+      inCodeBlock = false;
+      yield "\n";
+    } else if (inCodeBlock) {
+      yield line;
+    }
+  }
+}
+
+const files = glob.sync(globString, { recursive: true });
+
+for (const file of files.filter((file) => !excludedFiles.includes(file))) {
+  const lines = [];
+  const data = fs.readFileSync(file, "utf-8");
+  const fileLines = data.split("\n");
+
+  for (const line of yieldLines(fileLines, "```", "```")) {
+    lines.push(line);
+  }
+
+  if (lines.length > 0) {
+    const fileName = path.basename(file, ".md");
+    const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
+    console.log(outPath)
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
+  }
+}

docs/test/md_testing.py

@@ -0,0 +1,41 @@
+import glob
+from typing import Iterator
+from pathlib import Path
+
+excluded_files = [
+    "../src/fts.md",
+    "../src/embedding.md",
+    "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
+    "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+    "../src/examples/youtube_transcript_bot_with_nodejs.md"
+]
+
+python_prefix = "py"
+python_file = ".py"
+python_folder = "python"
+glob_string = "../src/**/*.md"
+
+def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
+    in_code_block = False
+    # Python code has strict indentation
+    strip_length = 0
+    for line in lines:
+        if line.strip().startswith(prefix + python_prefix):
+            in_code_block = True
+            strip_length = len(line) - len(line.lstrip())
+        elif in_code_block and line.strip().startswith(suffix):
+            in_code_block = False
+            yield "\n"
+        elif in_code_block:
+            yield line[strip_length:]
+
+for file in filter(lambda file: file not in excluded_files, glob.glob(glob_string, recursive=True)):
+    with open(file, "r") as f:
+        lines = list(yield_lines(iter(f), "```", "```"))
+
+    if len(lines) > 0:
+        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(out_path)
+        out_path.parent.mkdir(exist_ok=True, parents=True)
+        with open(out_path, "w") as out:
+            out.writelines(lines)

docs/test/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "lancedb-docs-test",
+  "version": "1.0.0",
+  "description": "",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "fs": "^0.0.1-security",
+    "glob": "^10.2.7",
+    "path": "^0.12.7",
+    "vectordb": "https://gitpkg.now.sh/lancedb/lancedb/node?main"
+  }
+}

docs/test/requirements.txt

@@ -0,0 +1,5 @@
+lancedb @ git+https://github.com/lancedb/lancedb.git#egg=subdir&subdirectory=python
+numpy
+pandas
+pylance
+duckdb

node/.eslintrc.js

@@ -12,5 +12,6 @@ module.exports = {
     sourceType: 'module'
   },
   rules: {
+    "@typescript-eslint/method-signature-style": "off",
   }
 }

node/.npmignore

@@ -0,0 +1,4 @@
+gen_test_data.py
+index.node
+dist/lancedb*.tgz
+vectordb*.tgz

node/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.5] - 2023-06-00
+
+### Added
+
+- Support for macOS X86
+
 ## [0.1.4] - 2023-06-03
 
 ### Added

node/README.md

@@ -8,15 +8,21 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
 
 ```javascript
 const lancedb = require('vectordb');
-const db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>');
-const table = await db.openTable('my_table');
-const query = await table.search([0.1, 0.3]).setLimit(20).execute();
+const db = await lancedb.connect('data/sample-lancedb');
+const table = await db.createTable("my_table",
+      [{ id: 1, vector: [0.1, 1.0], item: "foo", price: 10.0 },
+      { id: 2, vector: [3.9, 0.5], item: "bar", price: 20.0 }])
+const results = await table.search([0.1, 0.3]).limit(20).execute();
 console.log(results);
 ```
 
@@ -24,17 +30,33 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-The LanceDB javascript is built with npm:
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
+
+```bash
+npm test
+```
+
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript
 
 ```bash
 npm run tsc
 ```
 
-Run the tests with
-
-```bash
-npm test
-```
+### Fix lints
 
 To run the linter and have it automatically fix all errors
 
@@ -46,4 +68,4 @@ To build documentation
 
 ```bash
 npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
-```
+```

node/examples/js-transformers/index.js

@@ -0,0 +1,66 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+
+async function example() {
+
+    const lancedb = require('vectordb')
+
+    // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
+    const { pipeline } = await import('@xenova/transformers')
+    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+
+
+    // Create embedding function from pipeline which returns a list of vectors from batch
+    // sourceColumn is the name of the column in the data to be embedded
+    //
+    // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
+    const embed_fun = {}
+    embed_fun.sourceColumn = 'text'
+    embed_fun.embed = async function (batch) {
+        let result = []
+        for (let text of batch) {
+            const res = await pipe(text, { pooling: 'mean', normalize: true })
+            result.push(Array.from(res['data']))
+        }
+        return (result)
+    }
+
+    // Link a folder and create a table with data
+    const db = await lancedb.connect('data/sample-lancedb')
+
+    const data = [
+        { id: 1, text: 'Cherry', type: 'fruit' },
+        { id: 2, text: 'Carrot', type: 'vegetable' },
+        { id: 3, text: 'Potato', type: 'vegetable' },
+        { id: 4, text: 'Apple', type: 'fruit' },
+        { id: 5, text: 'Banana', type: 'fruit' }
+    ]
+
+    const table = await db.createTable('food_table', data, "create", embed_fun)
+
+
+    // Query the table
+    const results = await table
+        .search("a sweet fruit to eat")
+        .metricType("cosine")
+        .limit(2)
+        .execute()
+    console.log(results.map(r => r.text))
+
+}
+
+example().then(_ => { console.log("Done!") })

node/examples/js-transformers/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "vectordb-example-js-transformers",
+  "version": "1.0.0",
+  "description": "Example for using transformers.js with lancedb",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@xenova/transformers": "^2.4.1",
+    "vectordb": "^0.1.12"
+  }
+
+}

node/examples/js-youtube-transcripts/index.js

@@ -0,0 +1,122 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+const lancedb = require('vectordb')
+const fs = require('fs/promises')
+const readline = require('readline/promises')
+const { stdin: input, stdout: output } = require('process')
+const { Configuration, OpenAIApi } = require('openai')
+
+// Download file from XYZ
+const INPUT_FILE_NAME = 'data/youtube-transcriptions_sample.jsonl';
+
+(async () => {
+  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+  const apiKey = process.env.OPENAI_API_KEY
+  // The embedding function will create embeddings for the 'context' column
+  const embedFunction = new lancedb.OpenAIEmbeddingFunction('context', apiKey)
+
+  // Connects to LanceDB
+  const db = await lancedb.connect('data/youtube-lancedb')
+
+  // Open the vectors table or create one if it does not exist
+  let tbl
+  if ((await db.tableNames()).includes('vectors')) {
+    tbl = await db.openTable('vectors', embedFunction)
+  } else {
+    tbl = await createEmbeddingsTable(db, embedFunction)
+  }
+
+  // Use OpenAI Completion API to generate and answer based on the context that LanceDB provides
+  const configuration = new Configuration({ apiKey })
+  const openai = new OpenAIApi(configuration)
+  const rl = readline.createInterface({ input, output })
+  try {
+    while (true) {
+      const query = await rl.question('Prompt: ')
+      const results = await tbl
+        .search(query)
+        .select(['title', 'text', 'context'])
+        .limit(3)
+        .execute()
+
+      // console.table(results)
+
+      const response = await openai.createCompletion({
+        model: 'text-davinci-003',
+        prompt: createPrompt(query, results),
+        max_tokens: 400,
+        temperature: 0,
+        top_p: 1,
+        frequency_penalty: 0,
+        presence_penalty: 0
+      })
+      console.log(response.data.choices[0].text)
+    }
+  } catch (err) {
+    console.log('Error: ', err)
+  } finally {
+    rl.close()
+  }
+  process.exit(1)
+})()
+
+async function createEmbeddingsTable (db, embedFunction) {
+  console.log(`Creating embeddings from ${INPUT_FILE_NAME}`)
+  // read the input file into a JSON array, skipping empty lines
+  const lines = (await fs.readFile(INPUT_FILE_NAME, 'utf-8'))
+    .toString()
+    .split('\n')
+    .filter(line => line.length > 0)
+    .map(line => JSON.parse(line))
+
+  const data = contextualize(lines, 20, 'video_id')
+  return await db.createTable('vectors', data, embedFunction)
+}
+
+// Each transcript has a small text column, we include previous transcripts in order to
+// have more context information when creating embeddings
+function contextualize (rows, contextSize, groupColumn) {
+  const grouped = []
+  rows.forEach(row => {
+    if (!grouped[row[groupColumn]]) {
+      grouped[row[groupColumn]] = []
+    }
+    grouped[row[groupColumn]].push(row)
+  })
+
+  const data = []
+  Object.keys(grouped).forEach(key => {
+    for (let i = 0; i < grouped[key].length; i++) {
+      const start = i - contextSize > 0 ? i - contextSize : 0
+      grouped[key][i].context = grouped[key].slice(start, i + 1).map(r => r.text).join(' ')
+    }
+    data.push(...grouped[key])
+  })
+  return data
+}
+
+// Creates a prompt by aggregating all relevant contexts
+function createPrompt (query, context) {
+  let prompt =
+      'Answer the question based on the context below.\n\n' +
+      'Context:\n'
+
+  // need to make sure our prompt is not larger than max size
+  prompt = prompt + context.map(c => c.context).join('\n\n---\n\n').substring(0, 3750)
+  prompt = prompt + `\n\nQuestion: ${query}\nAnswer:`
+  return prompt
+}

node/examples/js-youtube-transcripts/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "vectordb-example-js-openai",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "vectordb": "file:../..",
+    "openai": "^3.2.1"
+  }
+}

node/native.js

@@ -12,29 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;
 
-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
+    try {
+        // Might be developing locally, so try that. But don't expose that error
+        // to the user.
+        nativeLib = require("./index.node");
+    } catch {
+        throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
+
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+      
+  Source error: ${e}`);
     }
 }
 
-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;

node/package-lock.json

@@ -1,19 +1,32 @@
 {
   "name": "vectordb",
-  "version": "0.1.4",
+  "version": "0.1.18",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.4",
+      "version": "0.1.18",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
       "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
-        "apache-arrow": "^12.0.0"
+        "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^12.0.0",
+        "axios": "^1.4.0"
       },
       "devDependencies": {
+        "@neon-rs/cli": "^0.0.160",
         "@types/chai": "^4.3.4",
+        "@types/chai-as-promised": "^7.1.5",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
         "@types/sinon": "^10.0.15",
@@ -21,9 +34,10 @@
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
         "chai": "^4.3.7",
+        "chai-as-promised": "^7.1.1",
         "eslint": "^8.39.0",
         "eslint-config-standard-with-typescript": "^34.0.1",
-        "eslint-plugin-import": "^2.27.5",
+        "eslint-plugin-import": "^2.26.0",
         "eslint-plugin-n": "^15.7.0",
         "eslint-plugin-promise": "^6.1.1",
         "mocha": "^10.2.0",
@@ -35,6 +49,13 @@
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
         "typescript": "*"
+      },
+      "optionalDependencies": {
+        "@lancedb/vectordb-darwin-arm64": "0.1.18",
+        "@lancedb/vectordb-darwin-x64": "0.1.18",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.1.18",
+        "@lancedb/vectordb-linux-x64-gnu": "0.1.18",
+        "@lancedb/vectordb-win32-x64-msvc": "0.1.18"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -64,6 +85,97 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
     },
+    "node_modules/@cargo-messages/android-arm-eabi": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
+      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ]
+    },
+    "node_modules/@cargo-messages/darwin-arm64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
+      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@cargo-messages/darwin-x64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
+      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@cargo-messages/linux-arm-gnueabihf": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
+      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@cargo-messages/linux-x64-gnu": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
+      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@cargo-messages/win32-arm64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
+      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/@cargo-messages/win32-x64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
+      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -202,6 +314,89 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.18.tgz",
+      "integrity": "sha512-vu8MCFgaAAGmTJF+4RaoApROMpRVVgrCk+V9my4adAfWkkXbSmtxiDgiIwwL1VqdGb8UwzGn3kVbNW7idE1ojA==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-darwin-x64": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.18.tgz",
+      "integrity": "sha512-ZU30bd6frRyKJ515ow972PlqO2wIiNT4Ohor9+KbUwl/VKDyAwKOKG8cWhRJXTxk0k1oqpiJ6+Q28TcYJ0sSAw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.18.tgz",
+      "integrity": "sha512-2UroC026bUYwyciSRonYlXei0SoYbKgfWpozxYOu7GgBAV2CQQtaAPgWJTEl6ZiCNeBmBTx+j0h3+ydUfZA73Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.18.tgz",
+      "integrity": "sha512-DoQBskl22JAJFZh219ZOJ6o+f1niTZp0qRYngHa/kTIpLKzHWQ0OTtMCz32VBAjAsKjSLNxHE8rrT/S6tvS7KQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.18.tgz",
+      "integrity": "sha512-a/kUM3V6rWuXS80pPECYxKfCUAnq56Of/GPCvnAkpk9C9ldyX10iff4aA6DiPHjEk9V2ytqDfJKl9N3QcMLKLA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
+      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      },
+      "optionalDependencies": {
+        "@cargo-messages/android-arm-eabi": "0.0.160",
+        "@cargo-messages/darwin-arm64": "0.0.160",
+        "@cargo-messages/darwin-x64": "0.0.160",
+        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
+        "@cargo-messages/linux-x64-gnu": "0.0.160",
+        "@cargo-messages/win32-arm64-msvc": "0.0.160",
+        "@cargo-messages/win32-x64-msvc": "0.0.160"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -311,6 +506,15 @@
       "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==",
       "dev": true
     },
+    "node_modules/@types/chai-as-promised": {
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz",
+      "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/chai": "*"
+      }
+    },
     "node_modules/@types/command-line-args": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
@@ -787,24 +991,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/array.prototype.flatmap": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz",
-      "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.1.4",
-        "es-abstract": "^1.20.4",
-        "es-shim-unscopables": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/assertion-error": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
@@ -817,8 +1003,7 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
@@ -833,12 +1018,13 @@
       }
     },
     "node_modules/axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
       "dependencies": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "node_modules/balanced-match": {
@@ -960,6 +1146,18 @@
         "node": ">=4"
       }
     },
+    "node_modules/chai-as-promised": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+      "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+      "dev": true,
+      "dependencies": {
+        "check-error": "^1.0.2"
+      },
+      "peerDependencies": {
+        "chai": ">= 2.1.2 < 5"
+      }
+    },
     "node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -1057,7 +1255,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -1280,7 +1477,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -1633,25 +1829,23 @@
       }
     },
     "node_modules/eslint-plugin-import": {
-      "version": "2.27.5",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz",
-      "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==",
+      "version": "2.26.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz",
+      "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==",
       "dev": true,
       "dependencies": {
-        "array-includes": "^3.1.6",
-        "array.prototype.flat": "^1.3.1",
-        "array.prototype.flatmap": "^1.3.1",
-        "debug": "^3.2.7",
+        "array-includes": "^3.1.4",
+        "array.prototype.flat": "^1.2.5",
+        "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.7",
-        "eslint-module-utils": "^2.7.4",
+        "eslint-import-resolver-node": "^0.3.6",
+        "eslint-module-utils": "^2.7.3",
         "has": "^1.0.3",
-        "is-core-module": "^2.11.0",
+        "is-core-module": "^2.8.1",
         "is-glob": "^4.0.3",
         "minimatch": "^3.1.2",
-        "object.values": "^1.1.6",
-        "resolve": "^1.22.1",
-        "semver": "^6.3.0",
+        "object.values": "^1.1.5",
+        "resolve": "^1.22.0",
         "tsconfig-paths": "^3.14.1"
       },
       "engines": {
@@ -1662,12 +1856,12 @@
       }
     },
     "node_modules/eslint-plugin-import/node_modules/debug": {
-      "version": "3.2.7",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
       "dev": true,
       "dependencies": {
-        "ms": "^2.1.1"
+        "ms": "2.0.0"
       }
     },
     "node_modules/eslint-plugin-import/node_modules/doctrine": {
@@ -1682,14 +1876,11 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/eslint-plugin-import/node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
-      "dev": true,
-      "bin": {
-        "semver": "bin/semver.js"
-      }
+    "node_modules/eslint-plugin-import/node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "dev": true
     },
     "node_modules/eslint-plugin-n": {
       "version": "15.7.0",
@@ -2052,7 +2243,6 @@
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
       "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -2081,7 +2271,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -2955,7 +3144,6 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -2964,7 +3152,6 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -3258,6 +3445,15 @@
         "form-data": "^4.0.0"
       }
     },
+    "node_modules/openai/node_modules/axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
     "node_modules/optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -3409,6 +3605,11 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "node_modules/punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -3619,9 +3820,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
-      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -4501,6 +4702,55 @@
         }
       }
     },
+    "@cargo-messages/android-arm-eabi": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
+      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/darwin-arm64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
+      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/darwin-x64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
+      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/linux-arm-gnueabihf": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
+      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/linux-x64-gnu": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
+      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/win32-arm64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
+      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/win32-x64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
+      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
+      "dev": true,
+      "optional": true
+    },
     "@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -4601,6 +4851,56 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.18.tgz",
+      "integrity": "sha512-vu8MCFgaAAGmTJF+4RaoApROMpRVVgrCk+V9my4adAfWkkXbSmtxiDgiIwwL1VqdGb8UwzGn3kVbNW7idE1ojA==",
+      "optional": true
+    },
+    "@lancedb/vectordb-darwin-x64": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.18.tgz",
+      "integrity": "sha512-ZU30bd6frRyKJ515ow972PlqO2wIiNT4Ohor9+KbUwl/VKDyAwKOKG8cWhRJXTxk0k1oqpiJ6+Q28TcYJ0sSAw==",
+      "optional": true
+    },
+    "@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.18.tgz",
+      "integrity": "sha512-2UroC026bUYwyciSRonYlXei0SoYbKgfWpozxYOu7GgBAV2CQQtaAPgWJTEl6ZiCNeBmBTx+j0h3+ydUfZA73Q==",
+      "optional": true
+    },
+    "@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.18.tgz",
+      "integrity": "sha512-DoQBskl22JAJFZh219ZOJ6o+f1niTZp0qRYngHa/kTIpLKzHWQ0OTtMCz32VBAjAsKjSLNxHE8rrT/S6tvS7KQ==",
+      "optional": true
+    },
+    "@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.1.18",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.18.tgz",
+      "integrity": "sha512-a/kUM3V6rWuXS80pPECYxKfCUAnq56Of/GPCvnAkpk9C9ldyX10iff4aA6DiPHjEk9V2ytqDfJKl9N3QcMLKLA==",
+      "optional": true
+    },
+    "@neon-rs/cli": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
+      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
+      "dev": true,
+      "requires": {
+        "@cargo-messages/android-arm-eabi": "0.0.160",
+        "@cargo-messages/darwin-arm64": "0.0.160",
+        "@cargo-messages/darwin-x64": "0.0.160",
+        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
+        "@cargo-messages/linux-x64-gnu": "0.0.160",
+        "@cargo-messages/win32-arm64-msvc": "0.0.160",
+        "@cargo-messages/win32-x64-msvc": "0.0.160"
+      }
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4703,6 +5003,15 @@
       "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==",
       "dev": true
     },
+    "@types/chai-as-promised": {
+      "version": "7.1.5",
+      "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz",
+      "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==",
+      "dev": true,
+      "requires": {
+        "@types/chai": "*"
+      }
+    },
     "@types/command-line-args": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
@@ -5038,18 +5347,6 @@
         "es-shim-unscopables": "^1.0.0"
       }
     },
-    "array.prototype.flatmap": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz",
-      "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==",
-      "dev": true,
-      "requires": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.1.4",
-        "es-abstract": "^1.20.4",
-        "es-shim-unscopables": "^1.0.0"
-      }
-    },
     "assertion-error": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
@@ -5059,8 +5356,7 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
     "available-typed-arrays": {
       "version": "1.0.5",
@@ -5069,12 +5365,13 @@
       "dev": true
     },
     "axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
       "requires": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "balanced-match": {
@@ -5172,6 +5469,15 @@
         "type-detect": "^4.0.5"
       }
     },
+    "chai-as-promised": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz",
+      "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==",
+      "dev": true,
+      "requires": {
+        "check-error": "^1.0.2"
+      }
+    },
     "chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -5245,7 +5551,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
       }
@@ -5412,8 +5717,7 @@
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
     },
     "diff": {
       "version": "4.0.2",
@@ -5707,35 +6011,33 @@
       }
     },
     "eslint-plugin-import": {
-      "version": "2.27.5",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz",
-      "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==",
+      "version": "2.26.0",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz",
+      "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==",
       "dev": true,
       "requires": {
-        "array-includes": "^3.1.6",
-        "array.prototype.flat": "^1.3.1",
-        "array.prototype.flatmap": "^1.3.1",
-        "debug": "^3.2.7",
+        "array-includes": "^3.1.4",
+        "array.prototype.flat": "^1.2.5",
+        "debug": "^2.6.9",
         "doctrine": "^2.1.0",
-        "eslint-import-resolver-node": "^0.3.7",
-        "eslint-module-utils": "^2.7.4",
+        "eslint-import-resolver-node": "^0.3.6",
+        "eslint-module-utils": "^2.7.3",
         "has": "^1.0.3",
-        "is-core-module": "^2.11.0",
+        "is-core-module": "^2.8.1",
         "is-glob": "^4.0.3",
         "minimatch": "^3.1.2",
-        "object.values": "^1.1.6",
-        "resolve": "^1.22.1",
-        "semver": "^6.3.0",
+        "object.values": "^1.1.5",
+        "resolve": "^1.22.0",
         "tsconfig-paths": "^3.14.1"
       },
       "dependencies": {
         "debug": {
-          "version": "3.2.7",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
-          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
           "dev": true,
           "requires": {
-            "ms": "^2.1.1"
+            "ms": "2.0.0"
           }
         },
         "doctrine": {
@@ -5747,10 +6049,10 @@
             "esutils": "^2.0.2"
           }
         },
-        "semver": {
-          "version": "6.3.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
+        "ms": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+          "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
           "dev": true
         }
       }
@@ -5985,8 +6287,7 @@
     "follow-redirects": {
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
     },
     "for-each": {
       "version": "0.3.3",
@@ -6001,7 +6302,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -6615,14 +6915,12 @@
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
     },
     "mime-types": {
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "requires": {
         "mime-db": "1.52.0"
       }
@@ -6848,6 +7146,17 @@
       "requires": {
         "axios": "^0.26.0",
         "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.26.1",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+          "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+          "dev": true,
+          "requires": {
+            "follow-redirects": "^1.14.8"
+          }
+        }
       }
     },
     "optionator": {
@@ -6956,6 +7265,11 @@
       "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
       "dev": true
     },
+    "proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -7078,9 +7392,9 @@
       }
     },
     "semver": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
-      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
       "dev": true,
       "requires": {
         "lru-cache": "^6.0.0"

node/package.json

@@ -1,16 +1,18 @@
 {
   "name": "vectordb",
-  "version": "0.1.4",
+  "version": "0.1.18",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
-    "test": "mocha -recursive dist/test",
+    "test": "npm run tsc && mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/"
+    "clean": "rm -rf node_modules *.node dist/",
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
   },
   "repository": {
     "type": "git",
@@ -25,7 +27,9 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@neon-rs/cli": "^0.0.160",
     "@types/chai": "^4.3.4",
+    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
     "@types/sinon": "^10.0.15",
@@ -33,9 +37,10 @@
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
     "chai": "^4.3.7",
+    "chai-as-promised": "^7.1.1",
     "eslint": "^8.39.0",
     "eslint-config-standard-with-typescript": "^34.0.1",
-    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
@@ -50,6 +55,33 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
-    "apache-arrow": "^12.0.0"
+    "@neon-rs/load": "^0.0.74",
+    "apache-arrow": "^12.0.0",
+    "axios": "^1.4.0"
+  },
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
+      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
+      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
+      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
+    }
+  },
+  "optionalDependencies": {
+    "@lancedb/vectordb-darwin-arm64": "0.1.18",
+    "@lancedb/vectordb-darwin-x64": "0.1.18",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.1.18",
+    "@lancedb/vectordb-linux-x64-gnu": "0.1.18",
+    "@lancedb/vectordb-win32-x64-msvc": "0.1.18"
   }
 }

node/src/embedding/embedding_function.ts

@@ -26,3 +26,9 @@ export interface EmbeddingFunction<T> {
      */
   embed: (data: T[]) => Promise<number[][]>
 }
+
+export function isEmbeddingFunction<T> (value: any): value is EmbeddingFunction<T> {
+  return Object.keys(value).length === 2 &&
+      typeof value.sourceColumn === 'string' &&
+      typeof value.embed === 'function'
+}

node/src/index.ts

@@ -14,47 +14,228 @@
 
 import {
   RecordBatchFileWriter,
-  type Table as ArrowTable,
-  tableFromIPC,
-  Vector
+  type Table as ArrowTable
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
+import { RemoteConnection } from './remote'
+import { Query } from './query'
+import { isEmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
 
+export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
+export interface AwsCredentials {
+  accessKeyId: string
+
+  secretKey: string
+
+  sessionToken?: string
+}
+
+export interface ConnectionOptions {
+  uri: string
+
+  awsCredentials?: AwsCredentials
+
+  // API key for the remote connections
+  apiKey?: string
+  // Region to connect
+  region?: string
+
+  // override the host for the remote connections
+  hostOverride?: string
+}
+
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
-export async function connect (uri: string): Promise<Connection> {
-  const db = await databaseNew(uri)
-  return new Connection(db, uri)
+export async function connect (uri: string): Promise<Connection>
+export async function connect (opts: Partial<ConnectionOptions>): Promise<Connection>
+export async function connect (arg: string | Partial<ConnectionOptions>): Promise<Connection> {
+  let opts: ConnectionOptions
+  if (typeof arg === 'string') {
+    opts = { uri: arg }
+  } else {
+    // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
+    opts = Object.assign({
+      uri: '',
+      awsCredentials: undefined,
+      apiKey: undefined,
+      region: 'us-west-2'
+    }, arg)
+  }
+
+  if (opts.uri.startsWith('db://')) {
+    // Remote connection
+    return new RemoteConnection(opts)
+  }
+  const db = await databaseNew(opts.uri)
+  return new LocalConnection(db, opts)
+}
+
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export interface Connection {
+  uri: string
+
+  tableNames(): Promise<string[]>
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
+
+  createTableArrow(name: string, table: ArrowTable): Promise<Table>
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  dropTable(name: string): Promise<void>
+
+}
+
+/**
+ * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+ */
+export interface Table<T = number[]> {
+  name: string
+
+  /**
+   * Creates a search query to find the nearest neighbors of the given search term
+   * @param query The query search term
+   */
+  search: (query: T) => Query<T>
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  add: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Insert records into this Table, replacing its contents.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Create an ANN index on this Table vector index.
+   *
+   * @param indexParams The parameters of this Index, @see VectorIndexParams.
+   */
+  createIndex: (indexParams: VectorIndexParams) => Promise<any>
+
+  /**
+   * Returns the number of rows in this table.
+   */
+  countRows: () => Promise<number>
+
+  /**
+   * Delete rows from this table.
+   *
+   * This can be used to delete a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause. The
+   *                filter must not be empty.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [1, 2]},
+   *    {id: 2, vector: [3, 4]},
+   *    {id: 3, vector: [5, 6]},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   * await tbl.delete("id = 2")
+   * await tbl.countRows() // Returns 2
+   * ```
+   *
+   * If you have a list of values to delete, you can combine them into a
+   * stringified list and use the `IN` operator:
+   *
+   * ```ts
+   * const to_remove = [1, 5];
+   * await tbl.delete(`id IN (${to_remove.join(",")})`)
+   * await tbl.countRows() // Returns 1
+   * ```
+   */
+  delete: (filter: string) => Promise<void>
 }
 
 /**
  * A connection to a LanceDB database.
  */
-export class Connection {
-  private readonly _uri: string
+export class LocalConnection implements Connection {
+  private readonly _options: ConnectionOptions
   private readonly _db: any
 
-  constructor (db: any, uri: string) {
-    this._uri = uri
+  constructor (db: any, options: ConnectionOptions) {
+    this._options = options
     this._db = db
   }
 
   get uri (): string {
-    return this._uri
+    return this._options.uri
   }
 
   /**
-     * Get the names of all tables in the database.
-     */
+   * Get the names of all tables in the database.
+   */
   async tableNames (): Promise<string[]> {
     return databaseTableNames.call(this._db)
   }
@@ -65,6 +246,7 @@ export class Connection {
    * @param name The name of the table.
    */
   async openTable (name: string): Promise<Table>
+
   /**
    * Open a table in the database.
    *
@@ -72,37 +254,43 @@ export class Connection {
    * @param embeddings An embedding function to use on this Table
    */
   async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
     const tbl = await databaseOpenTable.call(this._db, name)
     if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      return new Table(tbl, name)
+      return new LocalTable(tbl, name, this._options)
     }
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   */
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
+    let writeOptions: WriteOptions = new DefaultWriteOptions()
+    if (opt !== undefined && isWriteOptions(opt)) {
+      writeOptions = opt
+    } else if (optsOrEmbedding !== undefined && isWriteOptions(optsOrEmbedding)) {
+      writeOptions = optsOrEmbedding
+    }
+
+    let embeddings: undefined | EmbeddingFunction<T>
+    if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
+      embeddings = optsOrEmbedding
+    }
+    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), writeOptions.writeMode?.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      createArgs.push(this._options.awsCredentials.accessKeyId)
+      createArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        createArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+
+    const tbl = await tableCreate.call(...createArgs)
 
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings))
     if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      return new Table(tbl, name)
+      return new LocalTable(tbl, name, this._options)
     }
   }
 
@@ -111,24 +299,35 @@ export class Connection {
     await tableCreate.call(this._db, name, Buffer.from(await writer.toUint8Array()))
     return await this.openTable(name)
   }
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  async dropTable (name: string): Promise<void> {
+    await databaseDropTable.call(this._db, name)
+  }
 }
 
-export class Table<T = number[]> {
+export class LocalTable<T = number[]> implements Table<T> {
   private readonly _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _options: ConnectionOptions
 
-  constructor (tbl: any, name: string)
+  constructor (tbl: any, name: string, options: ConnectionOptions)
   /**
    * @param tbl
    * @param name
+   * @param options
    * @param embeddings An embedding function to use when interacting with this table
    */
-  constructor (tbl: any, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings: EmbeddingFunction<T>)
+  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._name = name
     this._embeddings = embeddings
+    this._options = options
   }
 
   get name (): string {
@@ -140,7 +339,7 @@ export class Table<T = number[]> {
    * @param query The query search term
    */
   search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+    return new Query(query, this._tbl, this._embeddings)
   }
 
   /**
@@ -150,7 +349,15 @@ export class Table<T = number[]> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString())
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+    return tableAdd.call(...callArgs)
   }
 
   /**
@@ -160,6 +367,14 @@ export class Table<T = number[]> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
     return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
   }
 
@@ -173,14 +388,25 @@ export class Table<T = number[]> {
   }
 
   /**
-   * @deprecated Use [Table.createIndex]
+   * Returns the number of rows in this table.
    */
-  async create_index (indexParams: VectorIndexParams): Promise<any> {
-    return await this.createIndex(indexParams)
+  async countRows (): Promise<number> {
+    return tableCountRows.call(this._tbl)
+  }
+
+  /**
+   * Delete rows from this table.
+   *
+   * @param filter A filter in the same format used by a sql WHERE clause.
+   */
+  async delete (filter: string): Promise<void> {
+    return tableDelete.call(this._tbl, filter)
   }
 }
 
-interface IvfPQIndexConfig {
+/// Config to build IVF_PQ index.
+///
+export interface IvfPQIndexConfig {
   /**
    * The column to be indexed
    */
@@ -225,121 +451,43 @@ interface IvfPQIndexConfig {
    */
   max_opq_iters?: number
 
+  /**
+   * Replace an existing index with the same name if it exists.
+   */
+  replace?: boolean
+
   type: 'ivf_pq'
 }
 
 export type VectorIndexParams = IvfPQIndexConfig
 
 /**
- * A builder for nearest neighbor queries for LanceDB.
+ * Write mode for writing a table.
  */
-export class Query<T = number[]> {
-  private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
-  private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
-  private _filter?: string
-  private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
-
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
-    this._tbl = tbl
-    this._query = query
-    this._limit = 10
-    this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
-    this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
-  }
-
-  /***
-   * Sets the number of results that will be returned
-   * @param value number of results
-   */
-  limit (value: number): Query<T> {
-    this._limit = value
-    return this
-  }
-
-  /**
-   * Refine the results by reading extra elements and re-ranking them in memory.
-   * @param value refine factor to use in this query.
-   */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-   * The number of probes used. A higher number makes search more accurate but also slower.
-   * @param value The number of probes used.
-   */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-   * A filter statement to be applied to this query.
-   * @param value A filter in the same format used by a sql WHERE clause.
-   */
-  filter (value: string): Query<T> {
-    this._filter = value
-    return this
-  }
-
-  /** Return only the specified columns.
-   *
-   * @param value Only select the specified columns. If not specified, all columns will be returned.
-   */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
-  /**
-   * The MetricType used for this Query.
-   * @param value The metric to the. @see MetricType for the different options
-   */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-   * Execute the query and return the results as an Array of Objects
-   */
-  async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
-    const buffer = await tableSearch.call(this._tbl, this)
-    const data = tableFromIPC(buffer)
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
+export enum WriteMode {
+  /** Create a new {@link Table}. */
+  Create = 'create',
+  /** Overwrite the existing {@link Table} if presented. */
+  Overwrite = 'overwrite',
+  /** Append new data to the table. */
+  Append = 'append'
 }
 
-export enum WriteMode {
-  Overwrite = 'overwrite',
-  Append = 'append'
+/**
+ * Write options when creating a Table.
+ */
+export interface WriteOptions {
+  /** A {@link WriteMode} to use on this operation */
+  writeMode?: WriteMode
+}
+
+export class DefaultWriteOptions implements WriteOptions {
+  writeMode = WriteMode.Create
+}
+
+export function isWriteOptions (value: any): value is WriteOptions {
+  return Object.keys(value).length === 1 &&
+      (value.writeMode === undefined || typeof value.writeMode === 'string')
 }
 
 /**
@@ -354,5 +502,10 @@ export enum MetricType {
   /**
    * Cosine distance
    */
-  Cosine = 'cosine'
+  Cosine = 'cosine',
+
+  /**
+   * Dot product
+   */
+  Dot = 'dot'
 }

node/src/query.ts

@@ -0,0 +1,130 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Vector, tableFromIPC } from 'apache-arrow'
+import { type EmbeddingFunction } from './embedding/embedding_function'
+import { type MetricType } from '.'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { tableSearch } = require('../native.js')
+
+/**
+ * A builder for nearest neighbor queries for LanceDB.
+ */
+export class Query<T = number[]> {
+  private readonly _query: T
+  private readonly _tbl?: any
+  private _queryVector?: number[]
+  private _limit: number
+  private _refineFactor?: number
+  private _nprobes: number
+  private _select?: string[]
+  private _filter?: string
+  private _metricType?: MetricType
+  protected readonly _embeddings?: EmbeddingFunction<T>
+
+  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+    this._tbl = tbl
+    this._query = query
+    this._limit = 10
+    this._nprobes = 20
+    this._refineFactor = undefined
+    this._select = undefined
+    this._filter = undefined
+    this._metricType = undefined
+    this._embeddings = embeddings
+  }
+
+  /***
+     * Sets the number of results that will be returned
+     * @param value number of results
+     */
+  limit (value: number): Query<T> {
+    this._limit = value
+    return this
+  }
+
+  /**
+     * Refine the results by reading extra elements and re-ranking them in memory.
+     * @param value refine factor to use in this query.
+     */
+  refineFactor (value: number): Query<T> {
+    this._refineFactor = value
+    return this
+  }
+
+  /**
+     * The number of probes used. A higher number makes search more accurate but also slower.
+     * @param value The number of probes used.
+     */
+  nprobes (value: number): Query<T> {
+    this._nprobes = value
+    return this
+  }
+
+  /**
+     * A filter statement to be applied to this query.
+     * @param value A filter in the same format used by a sql WHERE clause.
+     */
+  filter (value: string): Query<T> {
+    this._filter = value
+    return this
+  }
+
+  where = this.filter
+
+  /** Return only the specified columns.
+     *
+     * @param value Only select the specified columns. If not specified, all columns will be returned.
+     */
+  select (value: string[]): Query<T> {
+    this._select = value
+    return this
+  }
+
+  /**
+     * The MetricType used for this Query.
+     * @param value The metric to the. @see MetricType for the different options
+     */
+  metricType (value: MetricType): Query<T> {
+    this._metricType = value
+    return this
+  }
+
+  /**
+     * Execute the query and return the results as an Array of Objects
+     */
+  async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    if (this._embeddings !== undefined) {
+      this._queryVector = (await this._embeddings.embed([this._query]))[0]
+    } else {
+      this._queryVector = this._query as number[]
+    }
+
+    const buffer = await tableSearch.call(this._tbl, this)
+    const data = tableFromIPC(buffer)
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}

node/src/remote/client.ts

@@ -0,0 +1,107 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios, { type AxiosResponse } from 'axios'
+
+import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
+
+export class HttpLancedbClient {
+  private readonly _url: string
+  private readonly _apiKey: () => string
+
+  public constructor (
+    url: string,
+    apiKey: string,
+    private readonly _dbName?: string
+  ) {
+    this._url = url
+    this._apiKey = () => apiKey
+  }
+
+  get uri (): string {
+    return this._url
+  }
+
+  public async search (
+    tableName: string,
+    vector: number[],
+    k: number,
+    nprobes: number,
+    refineFactor?: number,
+    columns?: string[],
+    filter?: string
+  ): Promise<ArrowTable<any>> {
+    const response = await axios.post(
+              `${this._url}/v1/table/${tableName}/query/`,
+              {
+                vector,
+                k,
+                nprobes,
+                refineFactor,
+                columns,
+                filter
+              },
+              {
+                headers: {
+                  'Content-Type': 'application/json',
+                  'x-api-key': this._apiKey(),
+                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+                },
+                responseType: 'arraybuffer',
+                timeout: 10000
+              }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+
+    const table = tableFromIPC(response.data)
+    return table
+  }
+
+  /**
+   * Sent GET request.
+   */
+  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
+    const response = await axios.get(
+      `${this._url}${path}`,
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this._apiKey()
+        },
+        params,
+        timeout: 10000
+      }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+    return response
+  }
+}

node/src/remote/index.ts

@@ -0,0 +1,168 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
+  type ConnectionOptions
+} from '../index'
+import { Query } from '../query'
+
+import { type Table as ArrowTable, Vector } from 'apache-arrow'
+import { HttpLancedbClient } from './client'
+
+/**
+ * Remote connection.
+ */
+export class RemoteConnection implements Connection {
+  private readonly _client: HttpLancedbClient
+  private readonly _dbName: string
+
+  constructor (opts: ConnectionOptions) {
+    if (!opts.uri.startsWith('db://')) {
+      throw new Error(`Invalid remote DB URI: ${opts.uri}`)
+    }
+    if (opts.apiKey === undefined || opts.region === undefined) {
+      throw new Error('API key and region are not supported for remote connections')
+    }
+
+    this._dbName = opts.uri.slice('db://'.length)
+    let server: string
+    if (opts.hostOverride === undefined) {
+      server = `https://${this._dbName}.${opts.region}.api.lancedb.com`
+    } else {
+      server = opts.hostOverride
+    }
+    this._client = new HttpLancedbClient(server, opts.apiKey, opts.hostOverride === undefined ? undefined : this._dbName)
+  }
+
+  get uri (): string {
+    // add the lancedb+ prefix back
+    return 'db://' + this._client.uri
+  }
+
+  async tableNames (): Promise<string[]> {
+    const response = await this._client.get('/v1/table/')
+    return response.data.tables
+  }
+
+  async openTable (name: string): Promise<Table>
+  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (embeddings !== undefined) {
+      return new RemoteTable(this._client, name, embeddings)
+    } else {
+      return new RemoteTable(this._client, name)
+    }
+  }
+
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    throw new Error('Not implemented')
+  }
+
+  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
+    throw new Error('Not implemented')
+  }
+
+  async dropTable (name: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+}
+
+export class RemoteQuery<T = number[]> extends Query<T> {
+  constructor (query: T, private readonly _client: HttpLancedbClient,
+    private readonly _name: string, embeddings?: EmbeddingFunction<T>) {
+    super(query, undefined, embeddings)
+  }
+
+  // TODO: refactor this to a base class + queryImpl pattern
+  async execute<T = Record<string, unknown>>(): Promise<T[]> {
+    const embeddings = this._embeddings
+    const query = (this as any)._query
+    let queryVector: number[]
+
+    if (embeddings !== undefined) {
+      queryVector = (await embeddings.embed([query]))[0]
+    } else {
+      queryVector = query as number[]
+    }
+
+    const data = await this._client.search(
+      this._name,
+      queryVector,
+      (this as any)._limit,
+      (this as any)._nprobes,
+      (this as any)._refineFactor,
+      (this as any)._select,
+      (this as any)._filter
+    )
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}
+
+// we are using extend until we have next next version release
+// Table and Connection has both been refactored to interfaces
+export class RemoteTable<T = number[]> implements Table<T> {
+  private readonly _client: HttpLancedbClient
+  private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _name: string
+
+  constructor (client: HttpLancedbClient, name: string)
+  constructor (client: HttpLancedbClient, name: string, embeddings: EmbeddingFunction<T>)
+  constructor (client: HttpLancedbClient, name: string, embeddings?: EmbeddingFunction<T>) {
+    this._client = client
+    this._name = name
+    this._embeddings = embeddings
+  }
+
+  get name (): string {
+    return this._name
+  }
+
+  search (query: T): Query<T> {
+    return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
+  }
+
+  async add (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+    throw new Error('Not implemented')
+  }
+
+  async countRows (): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async delete (filter: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+}

node/src/test/io.ts

@@ -18,26 +18,48 @@ import { describe } from 'mocha'
 import { assert } from 'chai'
 
 import * as lancedb from '../index'
+import { type ConnectionOptions } from '../index'
 
 describe('LanceDB S3 client', function () {
   if (process.env.TEST_S3_BASE_URL != null) {
     const baseUri = process.env.TEST_S3_BASE_URL
     it('should have a valid url', async function () {
-      const uri = `${baseUri}/valid_url`
-      const table = await createTestDB(uri, 2, 20)
-      const con = await lancedb.connect(uri)
-      assert.equal(con.uri, uri)
+      const opts = { uri: `${baseUri}/valid_url` }
+      const table = await createTestDB(opts, 2, 20)
+      const con = await lancedb.connect(opts)
+      assert.equal(con.uri, opts.uri)
 
       const results = await table.search([0.1, 0.3]).limit(5).execute()
       assert.equal(results.length, 5)
-    })
+    }).timeout(10_000)
+  } else {
+    describe.skip('Skip S3 test', function () {})
+  }
+
+  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
+    const baseUri = process.env.TEST_S3_BASE_URL
+    it('use custom credentials', async function () {
+      const opts: ConnectionOptions = {
+        uri: `${baseUri}/custom_credentials`,
+        awsCredentials: {
+          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
+          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
+        }
+      }
+      const table = await createTestDB(opts, 2, 20)
+      const con = await lancedb.connect(opts)
+      assert.equal(con.uri, opts.uri)
+
+      const results = await table.search([0.1, 0.3]).limit(5).execute()
+      assert.equal(results.length, 5)
+    }).timeout(10_000)
   } else {
     describe.skip('Skip S3 test', function () {})
   }
 })
 
-async function createTestDB (uri: string, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(uri)
+async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
+  const con = await lancedb.connect(opts)
 
   const data = []
   for (let i = 0; i < numRows; i++) {

node/src/test/test.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2023 LanceDB Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,11 +13,16 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
-import { assert } from 'chai'
 import { track } from 'temp'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type EmbeddingFunction, MetricType, Query } from '../index'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions } from '../index'
+
+const expect = chai.expect
+const assert = chai.assert
+chai.use(chaiAsPromised)
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -27,6 +32,22 @@ describe('LanceDB client', function () {
       assert.equal(con.uri, uri)
     })
 
+    it('should accept an options object', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect({ uri })
+      assert.equal(con.uri, uri)
+    })
+
+    it('should accept custom aws credentials', async function () {
+      const uri = await createTestDB()
+      const awsCredentials: AwsCredentials = {
+        accessKeyId: '',
+        secretKey: ''
+      }
+      const con = await lancedb.connect({ uri, awsCredentials })
+      assert.equal(con.uri, uri)
+    })
+
     it('should return the existing table names', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -64,13 +85,20 @@ describe('LanceDB client', function () {
       assert.equal(results[0].id, 1)
     })
 
-    it('uses a filter', async function () {
+    it('uses a filter / where clause', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 1)
+        assert.equal(results[0].id, 2)
+      }
+
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.1]).filter('id == 2').execute()
-      assert.equal(results.length, 1)
-      assert.equal(results[0].id, 2)
+      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
+      assertResults(results)
+      results = await table.search([0.1, 0.1]).where('id == 2').execute()
+      assertResults(results)
     })
 
     it('select only a subset of columns', async function () {
@@ -103,9 +131,32 @@ describe('LanceDB client', function () {
       const tableName = `vectors_${Math.floor(Math.random() * 100)}`
       const table = await con.createTable(tableName, data)
       assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 2)
+    })
 
-      const results = await table.search([0.1, 0.3]).execute()
-      assert.equal(results.length, 2)
+    it('use overwrite flag to overwrite existing table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      const tableName = 'overwrite'
+      await con.createTable(tableName, data, { writeMode: WriteMode.Create })
+
+      const newData = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 },
+        { id: 3, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
+
+      const table = await con.createTable(tableName, newData, { writeMode: WriteMode.Overwrite })
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 3)
     })
 
     it('appends records to an existing table ', async function () {
@@ -118,16 +169,14 @@ describe('LanceDB client', function () {
       ]
 
       const table = await con.createTable('vectors', data)
-      const results = await table.search([0.1, 0.3]).execute()
-      assert.equal(results.length, 2)
+      assert.equal(await table.countRows(), 2)
 
       const dataAdd = [
         { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
         { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
       ]
       await table.add(dataAdd)
-      const resultsAdd = await table.search([0.1, 0.3]).execute()
-      assert.equal(resultsAdd.length, 4)
+      assert.equal(await table.countRows(), 4)
     })
 
     it('overwrite all records in a table', async function () {
@@ -135,16 +184,25 @@ describe('LanceDB client', function () {
       const con = await lancedb.connect(uri)
 
       const table = await con.openTable('vectors')
-      const results = await table.search([0.1, 0.3]).execute()
-      assert.equal(results.length, 2)
+      assert.equal(await table.countRows(), 2)
 
       const dataOver = [
         { vector: [2.1, 2.2], price: 10, name: 'foo' },
         { vector: [3.1, 3.2], price: 50, name: 'bar' }
       ]
       await table.overwrite(dataOver)
-      const resultsAdd = await table.search([0.1, 0.3]).execute()
-      assert.equal(resultsAdd.length, 2)
+      assert.equal(await table.countRows(), 2)
+    })
+
+    it('can delete records from a table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.delete('price = 10')
+      assert.equal(await table.countRows(), 1)
     })
   })
 
@@ -153,8 +211,25 @@ describe('LanceDB client', function () {
       const uri = await createTestDB(32, 300)
       const con = await lancedb.connect(uri)
       const table = await con.openTable('vectors')
-      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 })
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
     }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
+
+    it('replace an existing index', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      // Replace should fail if the index already exists
+      await expect(table.createIndex({
+        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
+      })
+      ).to.be.rejectedWith('LanceError(Index)')
+
+      // Default replace = true
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(50_000)
   })
 
   describe('when using a custom embedding function', function () {
@@ -184,7 +259,7 @@ describe('LanceDB client', function () {
         { price: 10, name: 'foo' },
         { price: 50, name: 'bar' }
       ]
-      const table = await con.createTable('vectors', data, embeddings)
+      const table = await con.createTable('vectors', data, embeddings, { writeMode: WriteMode.Create })
       const results = await table.search('foo').execute()
       assert.equal(results.length, 2)
     })
@@ -193,7 +268,7 @@ describe('LanceDB client', function () {
 
 describe('Query object', function () {
   it('sets custom parameters', async function () {
-    const query = new Query(undefined, [0.1, 0.3])
+    const query = new Query([0.1, 0.3])
       .limit(1)
       .metricType(MetricType.Cosine)
       .refineFactor(100)
@@ -223,3 +298,39 @@ async function createTestDB (numDimensions: number = 2, numRows: number = 2): Pr
   await con.createTable('vectors', data)
   return dir
 }
+
+describe('Drop table', function () {
+  it('drop a table', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    await con.createTable('t1', data)
+    await con.createTable('t2', data)
+
+    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
+
+    await con.dropTable('t1')
+    assert.deepEqual(await con.tableNames(), ['t2'])
+  })
+})
+
+describe('WriteOptions', function () {
+  context('#isWriteOptions', function () {
+    it('should not match empty object', function () {
+      assert.equal(isWriteOptions({}), false)
+    })
+    it('should match write options', function () {
+      assert.equal(isWriteOptions({ writeMode: WriteMode.Create }), true)
+    })
+    it('should match undefined write mode', function () {
+      assert.equal(isWriteOptions({ writeMode: undefined }), true)
+    })
+    it('should match default write options', function () {
+      assert.equal(isWriteOptions(new DefaultWriteOptions()), true)
+    })
+  })
+})

python/.bumpversion.cfg

@@ -0,0 +1,8 @@
+[bumpversion]
+current_version = 0.1.15
+commit = True
+message = [python] Bump version: {current_version} → {new_version}
+tag = True
+tag_name = python-v{new_version}
+
+[bumpversion:file:pyproject.toml]

python/README.md

@@ -0,0 +1,85 @@
+# LanceDB
+
+A Python library for [LanceDB](https://github.com/lancedb/lancedb).
+
+## Installation
+
+```bash
+pip install lancedb
+```
+
+## Usage
+
+### Basic Example
+
+```python
+import lancedb
+db = lancedb.connect('<PATH_TO_LANCEDB_DATASET>')
+table = db.open_table('my_table')
+results = table.search([0.1, 0.3]).limit(20).to_df()
+print(results)
+```
+
+
+## Development
+
+Create a virtual environment and activate it:
+
+```bash
+python -m venv venv
+. ./venv/bin/activate
+```
+
+Install the necessary packages:
+
+```bash
+python -m pip install .
+```
+
+To run the unit tests:
+
+```bash
+pytest
+```
+
+To run linter and automatically fix all errors:
+
+```bash
+black .
+isort .
+```
+
+If any packages are missing, install them with:
+
+```bash
+pip install <PACKAGE_NAME>
+```
+
+
+___
+For **Windows** users, there may be errors when installing packages, so these commands may be helpful:
+
+Activate the virtual environment:
+```bash
+. .\venv\Scripts\activate
+```
+
+You may need to run the installs separately:
+```bash
+pip install -e .[tests]
+pip install -e .[dev]
+```
+
+
+`tantivy` requires `rust` to be installed, so install it with `conda`, as it doesn't support windows installation:
+```bash
+pip install wheel
+pip install cargo
+conda install rust
+pip install tantivy
+```
+
+To run the unit tests:
+```bash
+pytest
+```

python/lancedb/__init__.py

@@ -11,19 +11,53 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from .db import URI, LanceDBConnection
+from typing import Optional
+
+from .db import URI, DBConnection, LanceDBConnection
+from .remote.db import RemoteDBConnection
+from .schema import vector
 
 
-def connect(uri: URI) -> LanceDBConnection:
-    """Connect to a LanceDB instance at the given URI
+def connect(
+    uri: URI,
+    *,
+    api_key: Optional[str] = None,
+    region: str = "us-west-2",
+    host_override: Optional[str] = None,
+) -> DBConnection:
+    """Connect to a LanceDB database.
 
     Parameters
     ----------
     uri: str or Path
         The uri of the database.
+    api_token: str, optional
+        If presented, connect to LanceDB cloud.
+        Otherwise, connect to a database on file system or cloud storage.
+
+    Examples
+    --------
+
+    For a local directory, provide a path for the database:
+
+    >>> import lancedb
+    >>> db = lancedb.connect("~/.lancedb")
+
+    For object storage, use a URI prefix:
+
+    >>> db = lancedb.connect("s3://my-bucket/lancedb")
+
+    Connect to LancdDB cloud:
+
+    >>> db = lancedb.connect("db://my_database", api_key="ldb_...")
 
     Returns
     -------
-    A connection to a LanceDB database.
+    conn : DBConnection
+        A connection to a LanceDB database.
     """
+    if isinstance(uri, str) and uri.startswith("db://"):
+        if api_key is None:
+            raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
+        return RemoteDBConnection(uri, api_key, region, host_override)
     return LanceDBConnection(uri)

python/lancedb/common.py

@@ -23,3 +23,13 @@ URI = Union[str, Path]
 # TODO support generator
 DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
+
+
+class Credential(str):
+    """Credential field"""
+
+    def __repr__(self) -> str:
+        return "********"
+
+    def __str__(self) -> str:
+        return "********"

python/lancedb/conftest.py

@@ -0,0 +1,16 @@
+import os
+
+import pytest
+
+# import lancedb so we don't have to in every example
+
+
+@pytest.fixture(autouse=True)
+def doctest_setup(monkeypatch, tmpdir):
+    # disable color for doctests so we don't have to include
+    # escape codes in docstrings
+    monkeypatch.setitem(os.environ, "NO_COLOR", "1")
+    # Explicitly set the column width
+    monkeypatch.setitem(os.environ, "COLUMNS", "80")
+    # Work in a temporary directory
+    monkeypatch.chdir(tmpdir)

python/lancedb/context.py

@@ -14,20 +14,109 @@ from __future__ import annotations
 
 import pandas as pd
 
+from .exceptions import MissingColumnError, MissingValueError
+
 
 def contextualize(raw_df: pd.DataFrame) -> Contextualizer:
     """Create a Contextualizer object for the given DataFrame.
-    Used to create context windows.
+
+    Used to create context windows. Context windows are rolling subsets of text
+    data.
+
+    The input text column should already be separated into rows that will be the
+    unit of the window. So to create a context window over tokens, start with
+    a DataFrame with one token per row. To create a context window over sentences,
+    start with a DataFrame with one sentence per row.
+
+    Examples
+    --------
+    >>> from lancedb.context import contextualize
+    >>> import pandas as pd
+    >>> data = pd.DataFrame({
+    ...    'token': ['The', 'quick', 'brown', 'fox', 'jumped', 'over',
+    ...              'the', 'lazy', 'dog', 'I', 'love', 'sandwiches'],
+    ...    'document_id': [1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2]
+    ... })
+
+    ``window`` determines how many rows to include in each window. In our case
+    this how many tokens, but depending on the input data, it could be sentences,
+    paragraphs, messages, etc.
+
+    >>> contextualize(data).window(3).stride(1).text_col('token').to_df()
+                    token  document_id
+    0     The quick brown            1
+    1     quick brown fox            1
+    2    brown fox jumped            1
+    3     fox jumped over            1
+    4     jumped over the            1
+    5       over the lazy            1
+    6        the lazy dog            1
+    7          lazy dog I            1
+    8          dog I love            1
+    9   I love sandwiches            2
+    10    love sandwiches            2
+    >>> contextualize(data).window(7).stride(1).min_window_size(7).text_col('token').to_df()
+                                      token  document_id
+    0   The quick brown fox jumped over the            1
+    1  quick brown fox jumped over the lazy            1
+    2    brown fox jumped over the lazy dog            1
+    3        fox jumped over the lazy dog I            1
+    4       jumped over the lazy dog I love            1
+    5   over the lazy dog I love sandwiches            1
+
+    ``stride`` determines how many rows to skip between each window start. This can
+    be used to reduce the total number of windows generated.
+
+    >>> contextualize(data).window(4).stride(2).text_col('token').to_df()
+                        token  document_id
+    0     The quick brown fox            1
+    2   brown fox jumped over            1
+    4    jumped over the lazy            1
+    6          the lazy dog I            1
+    8   dog I love sandwiches            1
+    10        love sandwiches            2
+
+    ``groupby`` determines how to group the rows. For example, we would like to have
+    context windows that don't cross document boundaries. In this case, we can
+    pass ``document_id`` as the group by.
+
+    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_df()
+                       token  document_id
+    0    The quick brown fox            1
+    2  brown fox jumped over            1
+    4   jumped over the lazy            1
+    6           the lazy dog            1
+    9      I love sandwiches            2
+
+    ``min_window_size`` determines the minimum size of the  context windows that are generated
+    This can be used to trim the last few context windows which have size less than
+    ``min_window_size``. By default context windows of size 1 are skipped.
+
+    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_df()
+                                 token  document_id
+    0  The quick brown fox jumped over            1
+    3     fox jumped over the lazy dog            1
+    6                     the lazy dog            1
+    9                I love sandwiches            2
+
+    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_df()
+                                 token  document_id
+    0  The quick brown fox jumped over            1
+    3     fox jumped over the lazy dog            1
+
     """
     return Contextualizer(raw_df)
 
 
 class Contextualizer:
+    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+
     def __init__(self, raw_df):
         self._text_col = None
         self._groupby = None
         self._stride = None
         self._window = None
+        self._min_window_size = 2
         self._raw_df = raw_df
 
     def window(self, window: int) -> Contextualizer:
@@ -75,17 +164,50 @@ class Contextualizer:
         self._text_col = text_col
         return self
 
+    def min_window_size(self, min_window_size: int) -> Contextualizer:
+        """Set the (optional) min_window_size size for the context window.
+
+        Parameters
+        ----------
+        min_window_size: int
+            The min_window_size.
+        """
+        self._min_window_size = min_window_size
+        return self
+
     def to_df(self) -> pd.DataFrame:
         """Create the context windows and return a DataFrame."""
 
+        if self._text_col not in self._raw_df.columns.tolist():
+            raise MissingColumnError(self._text_col)
+
+        if self._window is None or self._window < 1:
+            raise MissingValueError(
+                "The value of window is None or less than 1. Specify the "
+                "window size (number of rows to include in each window)"
+            )
+
+        if self._stride is None or self._stride < 1:
+            raise MissingValueError(
+                "The value of stride is None or less than 1. Specify the "
+                "stride (number of rows to skip between each window)"
+            )
+
         def process_group(grp):
             # For each group, create the text rolling window
+            # with values of size >= min_window_size
             text = grp[self._text_col].values
-            contexts = grp.iloc[: -self._window : self._stride, :].copy()
-            contexts[self._text_col] = [
-                " ".join(text[start_i : start_i + self._window])
-                for start_i in range(0, len(grp) - self._window, self._stride)
+            contexts = grp.iloc[:: self._stride, :].copy()
+            windows = [
+                " ".join(text[start_i : min(start_i + self._window, len(grp))])
+                for start_i in range(0, len(grp), self._stride)
+                if start_i + self._window <= len(grp)
+                or len(grp) - start_i >= self._min_window_size
             ]
+            # if last few rows dropped
+            if len(windows) < len(contexts):
+                contexts = contexts.iloc[: len(windows)]
+            contexts[self._text_col] = windows
             return contexts
 
         if self._groupby is None:

python/lancedb/db.py

@@ -14,24 +14,222 @@
 from __future__ import annotations
 
 import os
+from abc import ABC, abstractmethod
 from pathlib import Path
-import os
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 
+import pandas as pd
 import pyarrow as pa
 from pyarrow import fs
 
 from .common import DATA, URI
-from .table import LanceTable
-from .util import get_uri_scheme, get_uri_location
+from .table import LanceTable, Table
+from .util import fs_from_uri, get_uri_location, get_uri_scheme
 
 
-class LanceDBConnection:
+class DBConnection(ABC):
+    """An active LanceDB connection interface."""
+
+    @abstractmethod
+    def table_names(self) -> list[str]:
+        """List all table names in the database."""
+        pass
+
+    @abstractmethod
+    def create_table(
+        self,
+        name: str,
+        data: Optional[
+            Union[List[dict], dict, pd.DataFrame, pa.Table, Iterable[pa.RecordBatch]],
+        ] = None,
+        schema: Optional[pa.Schema] = None,
+        mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
+    ) -> Table:
+        """Create a [Table][lancedb.table.Table] in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        data: list, tuple, dict, pd.DataFrame; optional
+            The data to initialize the table. User must provide at least one of `data` or `schema`.
+        schema: pyarrow.Schema; optional
+            The schema of the table.
+        mode: str; default "create"
+            The mode to use when creating the table. Can be either "create" or "overwrite".
+            By default, if the table already exists, an exception is raised.
+            If you want to overwrite the table, use mode="overwrite".
+        on_bad_vectors: str, default "error"
+            What to do if any of the vectors are not the same size or contains NaNs.
+            One of "error", "drop", "fill".
+        fill_value: float
+            The value to use when filling vectors. Only used if on_bad_vectors="fill".
+
+        Returns
+        -------
+        LanceTable
+            A reference to the newly created table.
+
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
+        Examples
+        --------
+
+        Can create with list of tuples or dictionaries:
+
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
+        >>> db.create_table("my_table", data)
+        LanceTable(my_table)
+        >>> db["my_table"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        You can also pass a pandas DataFrame:
+
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
+        ...    "lat": [45.5, 40.1],
+        ...    "long": [-122.7, -74.1]
+        ... })
+        >>> db.create_table("table2", data)
+        LanceTable(table2)
+        >>> db["table2"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: double
+        long: double
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+        Data is converted to Arrow before being written to disk. For maximum
+        control over how data is saved, either provide the PyArrow schema to
+        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
+
+        >>> custom_schema = pa.schema([
+        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
+        ...   pa.field("lat", pa.float32()),
+        ...   pa.field("long", pa.float32())
+        ... ])
+        >>> db.create_table("table3", data, schema = custom_schema)
+        LanceTable(table3)
+        >>> db["table3"].head()
+        pyarrow.Table
+        vector: fixed_size_list<item: float>[2]
+          child 0, item: float
+        lat: float
+        long: float
+        ----
+        vector: [[[1.1,1.2],[0.2,1.8]]]
+        lat: [[45.5,40.1]]
+        long: [[-122.7,-74.1]]
+
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]]),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32())),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema)
+        LanceTable(table4)
+
+        """
+        raise NotImplementedError
+
+    def __getitem__(self, name: str) -> LanceTable:
+        return self.open_table(name)
+
+    def open_table(self, name: str) -> Table:
+        """Open a Lance Table in the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+
+        Returns
+        -------
+        A LanceTable object representing the table.
+        """
+        raise NotImplementedError
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        raise NotImplementedError
+
+
+class LanceDBConnection(DBConnection):
     """
     A connection to a LanceDB database.
+
+    Parameters
+    ----------
+    uri: str or Path
+        The root uri of the database.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> db = lancedb.connect("./.lancedb")
+    >>> db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2},
+    ...                                   {"vector": [0.5, 1.3], "b": 4}])
+    LanceTable(my_table)
+    >>> db.create_table("another_table", data=[{"vector": [0.4, 0.4], "b": 6}])
+    LanceTable(another_table)
+    >>> sorted(db.table_names())
+    ['another_table', 'my_table']
+    >>> len(db)
+    2
+    >>> db["my_table"]
+    LanceTable(my_table)
+    >>> "my_table" in db
+    True
+    >>> db.drop_table("my_table")
+    >>> db.drop_table("another_table")
     """
 
     def __init__(self, uri: URI):
-        is_local = isinstance(uri, Path) or get_uri_scheme(uri) == "file"
+        if not isinstance(uri, Path):
+            scheme = get_uri_scheme(uri)
+        is_local = isinstance(uri, Path) or scheme == "file"
         if is_local:
             if isinstance(uri, str):
                 uri = Path(uri)
@@ -39,6 +237,8 @@ class LanceDBConnection:
             Path(uri).mkdir(parents=True, exist_ok=True)
         self._uri = str(uri)
 
+        self._entered = False
+
     @property
     def uri(self) -> str:
         return self._uri
@@ -48,21 +248,26 @@ class LanceDBConnection:
 
         Returns
         -------
-        A list of table names.
+        list of str
+            A list of table names.
         """
         try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
+            filesystem, path = fs_from_uri(self.uri)
         except pa.ArrowInvalid:
-            raise NotImplementedError(
-                "Unsupported scheme: " + self.uri
-            )
+            raise NotImplementedError("Unsupported scheme: " + self.uri)
 
         try:
-            paths = filesystem.get_file_info(fs.FileSelector(get_uri_location(self.uri)))
+            paths = filesystem.get_file_info(
+                fs.FileSelector(get_uri_location(self.uri))
+            )
         except FileNotFoundError:
             # It is ok if the file does not exist since it will be created
             paths = []
-        tables = [os.path.splitext(file_info.base_name)[0] for file_info in paths if file_info.extension == 'lance']
+        tables = [
+            os.path.splitext(file_info.base_name)[0]
+            for file_info in paths
+            if file_info.extension == "lance"
+        ]
         return tables
 
     def __len__(self) -> int:
@@ -71,44 +276,33 @@ class LanceDBConnection:
     def __contains__(self, name: str) -> bool:
         return name in self.table_names()
 
-    def __getitem__(self, name: str) -> LanceTable:
-        return self.open_table(name)
-
     def create_table(
         self,
         name: str,
-        data: DATA = None,
+        data: Optional[Union[List[dict], dict, pd.DataFrame]] = None,
         schema: pa.Schema = None,
         mode: str = "create",
+        on_bad_vectors: str = "error",
+        fill_value: float = 0.0,
     ) -> LanceTable:
         """Create a table in the database.
 
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to insert into the table.
-        schema: pyarrow.Schema; optional
-            The schema of the table.
-        mode: str; default "create"
-            The mode to use when creating the table.
-            By default, if the table already exists, an exception is raised.
-            If you want to overwrite the table, use mode="overwrite".
-
-        Note
-        ----
-        The vector index won't be created by default.
-        To create the index, call the `create_index` method on the table.
-
-        Returns
-        -------
-        A LanceTable object representing the table.
+        See
+        ---
+        DBConnection.create_table
         """
-        if data is not None:
-            tbl = LanceTable.create(self, name, data, schema, mode=mode)
-        else:
-            tbl = LanceTable(self, name)
+        if mode.lower() not in ["create", "overwrite"]:
+            raise ValueError("mode must be either 'create' or 'overwrite'")
+
+        tbl = LanceTable.create(
+            self,
+            name,
+            data,
+            schema,
+            mode=mode,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+        )
         return tbl
 
     def open_table(self, name: str) -> LanceTable:
@@ -123,7 +317,7 @@ class LanceDBConnection:
         -------
         A LanceTable object representing the table.
         """
-        return LanceTable(self, name)
+        return LanceTable.open(self, name)
 
     def drop_table(self, name: str):
         """Drop a table from the database.
@@ -133,6 +327,6 @@ class LanceDBConnection:
         name: str
             The name of the table.
         """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        filesystem, path = fs_from_uri(self.uri)
         table_path = os.path.join(path, name + ".lance")
         filesystem.delete_dir(table_path)

python/lancedb/embeddings.py

@@ -29,7 +29,31 @@ def with_embeddings(
     wrap_api: bool = True,
     show_progress: bool = False,
     batch_size: int = 1000,
-):
+) -> pa.Table:
+    """Add a vector column to a table using the given embedding function.
+
+    The new columns will be called "vector".
+
+    Parameters
+    ----------
+    func : Callable
+        A function that takes a list of strings and returns a list of vectors.
+    data : pa.Table or pd.DataFrame
+        The data to add an embedding column to.
+    column : str, default "text"
+        The name of the column to use as input to the embedding function.
+    wrap_api : bool, default True
+        Whether to wrap the embedding function in a retry and rate limiter.
+    show_progress : bool, default False
+        Whether to show a progress bar.
+    batch_size : int, default 1000
+        The number of row values to pass to each call of the embedding function.
+
+    Returns
+    -------
+    pa.Table
+        The input table with a new column called "vector" containing the embeddings.
+    """
     func = EmbeddingFunction(func)
     if wrap_api:
         func = func.retry().rate_limit()

python/lancedb/exceptions.py

@@ -0,0 +1,22 @@
+"""Custom exception handling"""
+
+
+class MissingValueError(ValueError):
+    """Exception raised when a required value is missing."""
+
+    pass
+
+
+class MissingColumnError(KeyError):
+    """
+    Exception raised when a column name specified is not in
+    the  DataFrame object
+    """
+
+    def __init__(self, column_name):
+        self.column_name = column_name
+
+    def __str__(self):
+        return (
+            f"Error: Column '{self.column_name}' does not exist in the DataFrame object"
+        )