[python] Bump version: 0.1.16 → 0.2.0

BREAKING CHANGE: The `score` column has been renamed to `_distance` to
more accurately describe the semantics (smaller means closer / better).

---------

Co-authored-by: Lei Xu <lei@lancedb.com>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.13
+current_version = 0.1.19
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/docs_test.yml

@@ -81,7 +81,7 @@ jobs:
       run: |
         cd docs/test/node_modules/vectordb 
         npm ci
-        npm run build
+        npm run build-release
         npm run tsc
     - name: Create test files
       run: |

.github/workflows/make-release-commit.yml

@@ -25,31 +25,35 @@ jobs:
   bump-version:
     runs-on: ubuntu-latest
     steps:
-    - name: Check out main
+      - name: Check out main
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-      with:
+        with:
-        ref: main
+          ref: main
-        persist-credentials: false
+          persist-credentials: false
-        fetch-depth: 0
+          fetch-depth: 0
-        lfs: true
+          lfs: true
-    - name: Set git configs for bumpversion
+      - name: Set git configs for bumpversion
-      shell: bash
+        shell: bash
-      run: |
+        run: |
-        git config user.name 'Lance Release'
+          git config user.name 'Lance Release'
-        git config user.email 'lance-dev@lancedb.com'
+          git config user.email 'lance-dev@lancedb.com'
-    - name: Set up Python 3.10
+      - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+        uses: actions/setup-python@v4
-      with:
+        with:
-        python-version: "3.10"
+          python-version: "3.10"
-    - name: Bump version, create tag and commit
+      - name: Bump version, create tag and commit
-      run: |
+        run: |
-        pip install bump2version
+          pip install bump2version
-        bumpversion --verbose ${{ inputs.part }}
+          bumpversion --verbose ${{ inputs.part }}
-    - name: Push new version and tag
+      - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
+        if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
+        uses: ad-m/github-push-action@master
-      with:
+        with:
-        github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
-        branch: main
+          branch: main
-        tags: true
+          tags: true
+      - uses: ./.github/workflows/update_package_lock
+        if: ${{ inputs.dry_run }} == "false"
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
 

.github/workflows/node.yml

@@ -70,7 +70,7 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test
@@ -101,7 +101,7 @@ jobs:
         npm run tsc
         npm run build
         npm run pack-build
-        npm install --no-save ./dist/vectordb-*.tgz
+        npm install --no-save ./dist/lancedb-vectordb-*.tgz
         # Remove index.node to test with dependency installed
         rm index.node
     - name: Test

.github/workflows/npm-publish.yml

@@ -46,78 +46,87 @@ jobs:
       matrix:
         target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-    - name: Install system dependencies
+      - name: Install system dependencies
-      run: brew install protobuf
+        run: brew install protobuf
-    - name: Install npm dependencies
+      - name: Install npm dependencies
-      run: |
+        run: |
-        cd node
+          cd node
-        npm ci
+          npm ci
-    - name: Install rustup target
+      - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+        if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
+        run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
+      - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - name: Upload Darwin Artifacts
+      - name: Upload Darwin Artifacts
-      uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v3
-      with:
+        with:
-        name: darwin-native
+          name: native-darwin
-        path: |
+          path: |
-          node/dist/vectordb-darwin*.tgz
+            node/dist/lancedb-vectordb-darwin*.tgz
 
   node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     strategy:
       fail-fast: false
       matrix:
-        libc:
+        config:
-          - gnu
+          - arch: x86_64
-          # TODO: re-enable musl once we have refactored to pre-built containers
+            runner: ubuntu-latest
-          # Right now we have to build node from source which is too expensive.
+          - arch: aarch64
-          # - musl
+            runner: buildjet-4vcpu-ubuntu-2204-arm
-        arch:
-          - x86_64
-          # Building on aarch64 is too slow for now
-          # - aarch64
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v3
+        uses: actions/checkout@v3
-    - name: Change owner to root (for npm)
+      - name: Build Linux Artifacts
-      # The docker container is run as root, so we need the files to be owned by root
+        run: |
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
-      run: sudo chown -R root:root .
+      - name: Upload Linux Artifacts
-    - name: Set up QEMU
+        uses: actions/upload-artifact@v3
-      if: ${{ matrix.arch == 'aarch64' }}
+        with:
-      uses: docker/setup-qemu-action@v2
+          name: native-linux
-      with:
+          path: |
-        platforms: arm64
+            node/dist/lancedb-vectordb-linux*.tgz
-    - name: Build Linux GNU native node modules
+
-      if: ${{ matrix.libc == 'gnu' }}
+  node-windows:
-      run: |
+    runs-on: windows-2022
-        docker run \
+    # Only runs on tags that matches the make-release action
-          -v $(pwd):/io -w /io \
+    if: startsWith(github.ref, 'refs/tags/v')
-          rust:1.70-bookworm \
+    strategy:
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+      fail-fast: false
-    - name: Build musl Linux native node modules
+      matrix:
-      if: ${{ matrix.libc == 'musl' }}
+        target: [x86_64-pc-windows-msvc]
-      run: |
+    steps:
-        docker run --platform linux/arm64/v8 \
+      - name: Checkout
-          -v $(pwd):/io -w /io \
+        uses: actions/checkout@v3
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+      - name: Install Protoc v21.12
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+        working-directory: C:\
-    - name: Upload Linux Artifacts
+        run: |
-      uses: actions/upload-artifact@v3
+          New-Item -Path 'C:\protoc' -ItemType Directory
-      with:
+          Set-Location C:\protoc
-        name: linux-native
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
-        path: |
+          7z x protoc.zip
-          node/dist/vectordb-linux*.tgz
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: native-windows
+          path: |
+            node/dist/lancedb-vectordb-win32*.tgz
 
   release:
-    needs: [node, node-macos, node-linux]
+    needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -128,10 +137,27 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: 20
+          registry-url: 'https://registry.npmjs.org'
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
         run: |
-          for filename in */*.tgz; do
+          mv */*.tgz .
+          for filename in *.tgz; do
             npm publish $filename
           done
+
+  update-package-lock:
+    needs: [release]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

.github/workflows/python.yml

@@ -30,7 +30,7 @@ jobs:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
       run: |
-        pip install -e .
+        pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black isort
     - name: Black
@@ -59,7 +59,7 @@ jobs:
         python-version: "3.11"
     - name: Install lancedb
       run: |
-        pip install -e .
+        pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
         pip install pytest pytest-mock black
     - name: Black

.github/workflows/rust.yml

@@ -66,3 +66,24 @@ jobs:
         run: cargo build --all-features
       - name: Run tests
         run: cargo test --all-features
+  windows:
+    runs-on: windows-2022
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Run tests
+        run: |
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
+          cargo build
+          cargo test

.github/workflows/trigger-vectordb-recipes.yml

@@ -0,0 +1,26 @@
+name: Trigger vectordb-recipers workflow
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths:
+      - .github/workflows/trigger-vectordb-recipes.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Trigger vectordb-recipes workflow
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
+          script: |
+            const result = await github.rest.actions.createWorkflowDispatch({
+                owner: 'lancedb',
+                repo: 'vectordb-recipes',
+                workflow_id: 'examples-test.yml',
+                ref: 'main'
+            });
+            console.log(result);

.github/workflows/update_package_lock/action.yml

@@ -0,0 +1,33 @@
+name: update_package_lock
+description: "Update node's package.lock"
+
+inputs:
+  github_token:
+    required: true
+    description: "github token for the repo"
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Set git configs
+      shell: bash
+      run: |
+        git config user.name 'Lance Release'
+        git config user.email 'lance-dev@lancedb.com'
+    - name: Update package-lock.json file
+      working-directory: ./node
+      run: |
+        npm install
+        git add package-lock.json
+        git commit -m "Updating package-lock.json"
+      shell: bash
+    - name: Push changes
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ inputs.github_token }}
+        branch: main
+        tags: true

.github/workflows/update_package_lock_run.yml

@@ -0,0 +1,19 @@
+name: Update package-lock.json
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          persist-credentials: false
+          fetch-depth: 0
+          lfs: true
+      - uses: ./.github/workflows/update_package_lock
+        with:
+          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

Cargo.toml

@@ -6,9 +6,12 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-lance = "=0.5.5"
+lance = "=0.6.1"
-arrow-array = "42.0"
+arrow-array = "43.0"
-arrow-data = "42.0"
+arrow-data = "43.0"
-arrow-schema = "42.0"
+arrow-schema = "43.0"
-arrow-ipc = "42.0"
+arrow-ipc = "43.0"
+half = { "version" = "=2.2.1", default-features = false }
 object_store = "0.6.1"
+snafu = "0.7.4"
+

ci/build_linux_artifacts.sh

@@ -1,72 +1,19 @@
 #!/bin/bash
-# Builds the Linux artifacts (node binaries).
-# Usage: ./build_linux_artifacts.sh [target]
-# Targets supported: 
-# - x86_64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-musl
-# - x86_64-unknown-linux-musl
-
-# TODO: refactor this into a Docker container we can pull
-
 set -e
+ARCH=${1:-x86_64}
 
-setup_dependencies() {
+# We pass down the current user so that when we later mount the local files 
-    echo "Installing system dependencies..."
+# into the container, the files are accessible by the current user.
-    if [[ $1 == *musl ]]; then
+pushd ci/manylinux_node
-        # musllinux
+docker build \
-        apk add openssl-dev
+    -t lancedb-node-manylinux \
-    else
+    --build-arg="ARCH=$ARCH" \
-        # rust / debian
+    --build-arg="DOCKER_USER=$(id -u)" \
-        apt update
+    --progress=plain \
-        apt install -y libssl-dev protobuf-compiler
+    .
-    fi
+popd
-}
 
-install_node() {
+docker run \
-    echo "Installing node..."
+    -v $(pwd):/io -w /io \
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+    lancedb-node-manylinux \
-    source "$HOME"/.bashrc
+    bash ci/manylinux_node/build.sh $ARCH
-
-    if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
-    else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
-    fi
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
-    pushd node
-
-    npm ci
-    
-    if [[ $1 == *musl ]]; then
-        # This is needed for cargo to allow build cdylibs with musl
-        export RUSTFLAGS="-C target-feature=-crt-static"
-    fi
-
-    # Cargo can run out of memory while pulling dependencies, especially when running
-    # in QEMU. This is a workaround for that.
-    export CARGO_NET_GIT_FETCH_WITH_CLI=true
-
-    # We don't pass in target, since the native target here already matches
-    # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877
-    OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release
-    npm run pack-build
-
-    popd
-}
-
-TARGET=${1:-x86_64-unknown-linux-gnu}
-# Others:
-# aarch64-unknown-linux-gnu
-# x86_64-unknown-linux-musl
-# aarch64-unknown-linux-musl
-
-setup_dependencies $TARGET
-install_node $TARGET
-build_node_binary $TARGET

ci/build_windows_artifacts.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (node binaries).
+# Usage:  .\ci\build_windows_artifacts.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/ffi/node"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "node"
+    Write-Host "Building node library for $target"
+    npm run build-release -- --target $target
+    npm run pack-build -- --target $target
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

ci/manylinux_node/Dockerfile

@@ -0,0 +1,31 @@
+# Many linux dockerfile with Rust, Node, and Lance dependencies installed.
+# This container allows building the node modules native libraries in an 
+# environment with a very old glibc, so that we are compatible with a wide
+# range of linux distributions.
+ARG ARCH=x86_64
+
+FROM quay.io/pypa/manylinux2014_${ARCH}
+
+ARG ARCH=x86_64
+ARG DOCKER_USER=default_user
+
+# Install static openssl
+COPY install_openssl.sh install_openssl.sh
+RUN ./install_openssl.sh ${ARCH} > /dev/null
+
+# Protobuf is also installed as root.
+COPY install_protobuf.sh install_protobuf.sh
+RUN ./install_protobuf.sh ${ARCH}
+
+ENV DOCKER_USER=${DOCKER_USER}
+# Create a group and user
+RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+
+# We switch to the user to install Rust and Node, since those like to be 
+# installed at the user level.
+USER ${DOCKER_USER}
+
+COPY prepare_manylinux_node.sh prepare_manylinux_node.sh
+RUN cp /prepare_manylinux_node.sh $HOME/ && \
+    cd $HOME && \
+    ./prepare_manylinux_node.sh ${ARCH}

ci/manylinux_node/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
+set -e
+ARCH=${1:-x86_64}
+
+if [ "$ARCH" = "x86_64" ]; then
+    export OPENSSL_LIB_DIR=/usr/local/lib64/
+else 
+    export OPENSSL_LIB_DIR=/usr/local/lib/
+fi
+export OPENSSL_STATIC=1
+export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl
+
+source $HOME/.bashrc
+
+cd node
+npm ci
+npm run build-release
+npm run pack-build

ci/manylinux_node/install_openssl.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Builds openssl from source so we can statically link to it
+
+# this is to avoid the error we get with the system installation:
+# /usr/bin/ld: <library>: version node not found for symbol SSLeay@@OPENSSL_1.0.1
+# /usr/bin/ld: failed to set dynamic section sizes: Bad value
+set -e
+
+git clone -b OpenSSL_1_1_1u \
+    --single-branch \
+    https://github.com/openssl/openssl.git
+
+pushd openssl
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=linux-x86_64
+else
+    # gnu target
+    ARCH=linux-aarch64
+fi
+
+./Configure no-shared $ARCH
+
+make
+
+make install

ci/manylinux_node/install_protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Installs protobuf compiler. Should be run as root.
+set -e
+
+if [[ $1 == x86_64* ]]; then
+    ARCH=x86_64
+else
+    # gnu target
+    ARCH=aarch_64
+fi
+
+PB_REL=https://github.com/protocolbuffers/protobuf/releases
+PB_VERSION=23.1
+curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local

ci/manylinux_node/prepare_manylinux_node.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+install_node() {
+    echo "Installing node..."
+
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+
+    source "$HOME"/.bashrc
+
+    nvm install --no-progress 16
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+install_node
+install_rust

docs/mkdocs.yml

@@ -1,5 +1,6 @@
 site_name: LanceDB Docs
 repo_url: https://github.com/lancedb/lancedb
+edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
 docs_dir: src
 
@@ -10,6 +11,7 @@ theme:
   features:
     - content.code.copy
     - content.tabs.link
+    - content.action.edit
   icon:
     repo: fontawesome/brands/github
   custom_dir: overrides
@@ -57,9 +59,14 @@ nav:
 - Basics: basic.md
 - Embeddings: embedding.md
 - Python full-text search: fts.md
-- Python integrations:
+- Integrations:
   - Pandas and PyArrow: python/arrow.md
   - DuckDB: python/duckdb.md
+  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LangChain JS/TS 🦜️🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+  - Pydantic: python/pydantic.md
+  - Voxel51: integrations/voxel51.md
 - Python examples:
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
@@ -68,7 +75,10 @@ nav:
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
 - Javascript examples:
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
-- References:
+  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+
+- Guides:
+  - Tables: guides/tables.md
   - Vector Search: search.md
   - SQL filters: sql.md
   - Indexing: ann_indexes.md
@@ -78,3 +88,8 @@ nav:
 
 extra_css:
   - styles/global.css
+
+extra:
+  analytics:
+    provider: google
+    property: G-B7NFM40W74

docs/src/ann_indexes.md

@@ -94,7 +94,7 @@ There are a couple of parameters that can be used to fine-tune the search:
          .to_df()
      ```
      ```
-                                              vector       item       score
+                                              vector       item       _distance
      0  [0.44949695, 0.8444449, 0.06281311, 0.23338133...  item 1141  103.575333
      1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
      ```
@@ -109,9 +109,8 @@ There are a couple of parameters that can be used to fine-tune the search:
          .execute()
      ```
 
-The search will return the data requested in addition to the score of each item.
+The search will return the data requested in addition to the distance of each item.
 
-**Note:** The score is the distance between the query vector and the element. A lower number means that the result is more relevant.
 
 ### Filtering (where clause)
 
@@ -139,7 +138,7 @@ You can select the columns returned by the query using a select clause.
      tbl.search(np.random.random((1536))).select(["vector"]).to_df()
      ```
      ```
-     vector      score
+        vector                                             _distance
      0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
      1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
      ...

docs/src/basic.md

@@ -79,6 +79,18 @@ We'll cover the basics of using LanceDB on your local machine in this section.
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
+### Creating an empty table
+
+Sometimes you may not have the data to insert into the table at creation time.
+In this case, you can create an empty table and specify the schema.
+
+=== "Python"
+      ```python
+      import pyarrow as pa
+      schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
+      tbl = db.create_table("empty_table", schema=schema)
+      ```
+
 ## How to open an existing table
 
 Once created, you can open a table using the following code:
@@ -138,6 +150,49 @@ Once you've embedded the query, you can find its nearest neighbors using the fol
       const query = await tbl.search([100, 100]).limit(2).execute();
       ```
 
+## How to delete rows from a table
+
+Use the `delete()` method on tables to delete rows from a table. To choose
+which rows to delete, provide a filter that matches on the metadata columns.
+This can delete any number of rows that match the filter.
+
+=== "Python"
+      ```python
+      tbl.delete('item = "fizz"')
+      ```
+
+=== "Javascript"
+      ```javascript
+      await tbl.delete('item = "fizz"')
+      ```
+
+The deletion predicate is a SQL expression that supports the same expressions
+as the `where()` clause on a search. They can be as simple or complex as needed.
+To see what expressions are supported, see the [SQL filters](sql.md) section.
+
+
+=== "Python"
+
+      Read more: [lancedb.table.Table.delete][]
+
+=== "Javascript"
+
+      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+
+## How to remove a table
+
+Use the `drop_table()` method on the database to remove a table.
+
+=== "Python"
+      ```python
+      db.drop_table("my_table")
+      ```
+
+This permanently removes the table and is not recoverable, unlike deleting rows.
+By default, if the table does not exist an exception is raised. To suppress this,
+you can pass in `ignore_missing=True`.
+
+
 ## What's next
 
 This section covered the very basics of the LanceDB API.

docs/src/embedding.md

@@ -46,7 +46,7 @@ You can also use an external API like OpenAI to generate embeddings
 
         def embed_func(c):
             rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-        return [record["embedding"] for record in rs["data"]]
+            return [record["embedding"] for record in rs["data"]]
       ```
 
 === "Javascript"
@@ -126,7 +126,7 @@ belong in the same latent space and your results will be nonsensical.
 === "Javascript"
      ```javascript
       const results = await table
-        .search('What's the best pizza topping?')
+        .search("What's the best pizza topping?")
         .limit(10)
         .execute()
      ```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -0,0 +1,121 @@
+# Vector embedding search using TransformersJS
+
+## Embed and query data from LanceDB using TransformersJS
+
+<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
+
+This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
+
+
+### Setting up
+First, install the dependencies:
+```bash
+npm install vectordb
+npm i @xenova/transformers
+```
+
+We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
+
+Within our `index.js` file we will import the necessary libraries and define our model and database:
+
+```javascript
+const lancedb = require('vectordb')
+const { pipeline } = await import('@xenova/transformers')
+const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+```
+
+### Creating the embedding function
+
+Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
+
+```javascript
+// Define the function. `sourceColumn` is required for LanceDB to know
+// which column to use as input.
+const embed_fun = {}
+embed_fun.sourceColumn = 'text'
+embed_fun.embed = async function (batch) {
+    let result = []
+    // Given a batch of strings, we will use the `pipe` function to get
+    // the vector embedding of each string.
+    for (let text of batch) {
+        // 'mean' pooling and normalizing allows the embeddings to share the
+        // same length.
+        const res = await pipe(text, { pooling: 'mean', normalize: true })
+        result.push(Array.from(res['data']))
+    }
+    return (result)
+}
+```
+
+### Creating the database
+
+Now, we will create the LanceDB database and add the embedding function we defined earlier.
+
+```javascript
+// Link a folder and create a table with data
+const db = await lancedb.connect('data/sample-lancedb')
+
+// You can also import any other data, but make sure that you have a column
+// for the embedding function to use.
+const data = [
+    { id: 1, text: 'Cherry', type: 'fruit' },
+    { id: 2, text: 'Carrot', type: 'vegetable' },
+    { id: 3, text: 'Potato', type: 'vegetable' },
+    { id: 4, text: 'Apple', type: 'fruit' },
+    { id: 5, text: 'Banana', type: 'fruit' }
+]
+
+// Create the table with the embedding function
+const table = await db.createTable('food_table', data, "create", embed_fun)
+```
+
+### Performing the search
+
+Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
+
+```javascript
+// Query the table
+const results = await table
+    .search("a sweet fruit to eat")
+    .metricType("cosine")
+    .limit(2)
+    .execute()
+console.log(results.map(r => r.text))
+```
+```bash
+[ 'Banana', 'Cherry' ]
+```
+
+Output of `results`:
+```bash
+[
+  {
+    vector: Float32Array(384) [
+      -0.057455405592918396,
+      0.03617725893855095,
+      -0.0367760956287384,
+      ... 381 more items
+    ],
+    id: 5,
+    text: 'Banana',
+    type: 'fruit',
+    _distance: 0.4919965863227844
+  },
+  {
+    vector: Float32Array(384) [
+      0.0009714411571621895,
+      0.008223623037338257,
+      0.009571489877998829,
+      ... 381 more items
+    ],
+    id: 1,
+    text: 'Cherry',
+    type: 'fruit',
+    _distance: 0.5540297031402588
+  }
+]
+```
+
+### Wrapping it up
+
+In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/examples/youtube_transcript_bot.md

@@ -4,4 +4,10 @@
 
 <img id="splash" width="400" alt="youtube transcript search" src="https://user-images.githubusercontent.com/917119/236965568-def7394d-171c-45f2-939d-8edfeaadd88c.png">
 
+
+<a href="https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
+
+Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/youtube_bot/index.js)
+
+
 This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)

docs/src/guides/tables.md

@@ -0,0 +1,352 @@
+A Table is a collection of Records in a LanceDB Database.
+
+## Creating a LanceDB Table
+
+=== "Python"
+    ### LanceDB Connection
+
+    ```python
+    import lancedb
+    db = lancedb.connect("./.lancedb")
+    ```
+
+    LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
+
+    ### From list of tuples or dictionaries
+
+    ```python
+    import lancedb
+
+    db = lancedb.connect("./.lancedb")
+
+    data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
+            {"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1}]
+
+    db.create_table("my_table", data)
+
+    db["my_table"].head()
+    ```
+
+    !!! info "Note"
+        If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you can pass in mode="overwrite" to the createTable function.
+
+        ```python
+        db.create_table("name", data, mode="overwrite")
+        ```
+
+
+    ### From pandas DataFrame
+
+    ```python
+    import pandas as pd
+
+    data = pd.DataFrame({
+        "vector": [[1.1, 1.2], [0.2, 1.8]],
+        "lat": [45.5, 40.1],
+        "long": [-122.7, -74.1]
+    })
+
+    db.create_table("table2", data)
+
+    db["table2"].head() 
+    ```
+    !!! info "Note"
+        Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
+  
+    ```python
+    custom_schema = pa.schema([
+    pa.field("vector", pa.list_(pa.float32(), 2)),
+    pa.field("lat", pa.float32()),
+    pa.field("long", pa.float32())
+    ])
+
+    table = db.create_table("table3", data, schema=custom_schema)
+    ```
+
+    ### From Pydantic Models
+    LanceDB supports to create Apache Arrow Schema from a Pydantic BaseModel via pydantic_to_schema() method.
+
+    ```python
+    from lancedb.pydantic import vector, LanceModel
+
+    class Content(LanceModel):
+        movie_id: int
+        vector: vector(128)
+        genres: str
+        title: str
+        imdb_id: int
+            
+        @property
+        def imdb_url(self) -> str:
+            return f"https://www.imdb.com/title/tt{self.imdb_id}"
+
+    import pyarrow as pa
+    db = lancedb.connect("~/.lancedb")
+    table_name = "movielens_small"
+    table = db.create_table(table_name, schema=Content.to_arrow_schema())
+    ```
+
+    ### Using RecordBatch Iterator / Writing Large Datasets
+
+    It is recommended to use RecordBatch itertator to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`
+
+    ```python
+    import pyarrow as pa
+
+    def make_batches():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    schema = pa.schema([
+        pa.field("vector", pa.list_(pa.float32())),
+        pa.field("item", pa.utf8()),
+        pa.field("price", pa.float32()),
+    ])
+
+    db.create_table("table4", make_batches(), schema=schema)
+    ```
+
+    You can also use Pandas dataframe directly in the above example by converting it to `RecordBatch` object
+
+    ```python
+    import pandas as pd
+    import pyarrow as pa
+
+    df = pd.DataFrame({'vector': [[0,1], [2,3], [4,5],[6,7]],
+                        'month': [3, 5, 7, 9],
+                        'day': [1, 5, 9, 13],
+                        'n_legs': [2, 4, 5, 100],
+                        'animals': ["Flamingo", "Horse", "Brittle stars", "Centipede"]})
+
+    batch = pa.RecordBatch.from_pandas(df)
+    ```
+
+    ## Creating Empty Table
+    You can also create empty tables in python. Initialize it with schema and later ingest data into it.
+    
+    ```python
+    import lancedb
+    import pyarrow as pa
+  
+    schema = pa.schema(
+      [
+          pa.field("vector", pa.list_(pa.float32(), 2)),
+          pa.field("item", pa.string()),
+          pa.field("price", pa.float32()),
+      ])
+    tbl = db.create_table("table5", schema=schema)
+    data = [
+        {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+    ]
+    tbl.add(data=data)
+    ```
+
+    You can also use Pydantic to specify the schema
+
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, vector
+
+    class Model(LanceModel):
+          vector: vector(2)
+    
+    tbl = db.create_table("table5", schema=Model.to_arrow_schema())
+    ```
+    
+
+=== "Javascript/Typescript"
+
+    ### VectorDB Connection
+
+    ```javascript
+    const lancedb = require("vectordb");
+
+    const uri = "data/sample-lancedb";
+    const db = await lancedb.connect(uri);
+    ```
+
+    ### Creating a Table
+
+    You can create a LanceDB table in javascript using an array of records.
+
+    ```javascript
+    data
+    const tb = await db.createTable("my_table",
+                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+    ```
+
+    !!! info "Note"
+    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
+
+    ```javascript
+    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+    ```
+
+## Open existing tables
+
+If you forget the name of your table, you can always get a listing of all table names:
+
+
+=== "Python"
+    ### Get a list of existing Tables
+
+    ```python
+    print(db.table_names())
+    ```
+=== "Javascript/Typescript"
+
+    ```javascript
+    console.log(await db.tableNames());
+    ```
+
+Then, you can open any existing tables
+
+=== "Python"
+
+    ```python
+    tbl = db.open_table("my_table")
+    ```
+=== "Javascript/Typescript"
+
+    ```javascript
+    const tbl = await db.openTable("my_table");
+    ```
+
+## Adding to a Table
+After a table has been created, you can always add more data to it using
+
+=== "Python"
+    You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or a `Iterator[pa.RecordBatch]`. Here are some examples.
+
+    ### Adding Pandas DataFrame
+
+    ```python
+    df = pd.DataFrame([{"vector": [1.3, 1.4], "item": "fizz", "price": 100.0},
+                  {"vector": [9.5, 56.2], "item": "buzz", "price": 200.0}])
+    tbl.add(df)
+    ```
+
+    You can also add a large dataset batch in one go using pyArrow RecordBatch Iterator.
+
+    ### Adding RecordBatch Iterator
+    
+    ```python
+    import pyarrow as pa
+
+    def make_batches():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+    
+    tbl.add(make_batches())
+    ```
+  
+    The other arguments accepted:
+
+    | Name | Type | Description | Default |
+    |---|---|---|---|
+    | data | DATA | The data to insert into the table. | required |
+    | mode | str | The mode to use when writing the data. Valid values are "append" and "overwrite". | append |
+    | on_bad_vectors | str | What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". | drop |
+    | fill value | float | The value to use when filling vectors: Only used if on_bad_vectors="fill". | 0.0 |
+
+  
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.add([{vector: [1.3, 1.4], item: "fizz", price: 100.0},
+        {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
+    ```
+
+## Deleting from a Table
+
+Use the `delete()` method on tables to delete rows from a table. To choose which rows to delete, provide a filter that matches on the metadata columns. This can delete any number of rows that match the filter.
+
+=== "Python"
+
+    ```python
+    tbl.delete('item = "fizz"')
+    ```
+
+    ## Examples
+
+    ### Deleting row with specific column value
+
+    ```python
+    import lancedb
+    import pandas as pd
+
+    data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+    db = lancedb.connect("./.lancedb")
+    table = db.create_table("my_table", data)
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  2  [3.0, 4.0]
+    # 2  3  [5.0, 6.0]
+
+    table.delete("x = 2")
+    table.to_pandas()
+    #   x      vector
+    # 0  1  [1.0, 2.0]
+    # 1  3  [5.0, 6.0]
+    ```  
+
+    ### Delete from a list of values
+
+    ```python
+    to_remove = [1, 5]
+    to_remove = ", ".join(str(v) for v in to_remove)
+
+    table.delete(f"x IN ({to_remove})")
+    table.to_pandas()
+    #   x      vector
+    # 0  3  [5.0, 6.0]
+    ```
+  
+=== "Javascript/Typescript"
+
+    ```javascript
+    await tbl.delete('item = "fizz"')
+    ```
+
+    ### Deleting row with specific column value
+
+    ```javascript
+    const con = await lancedb.connect("./.lancedb")
+    const data = [
+      {id: 1, vector: [1, 2]},
+      {id: 2, vector: [3, 4]},
+      {id: 3, vector: [5, 6]},
+    ];
+    const tbl = await con.createTable("my_table", data)
+    await tbl.delete("id = 2")
+    await tbl.countRows() // Returns 2
+    ```
+
+    ### Delete from a list of values
+
+    ```javascript
+    const to_remove = [1, 5];
+    await tbl.delete(`id IN (${to_remove.join(",")})`)
+    await tbl.countRows() // Returns 1
+    ```
+
+## What's Next?
+
+Learn how to Query your tables and create indices

docs/src/index.md

@@ -69,4 +69,4 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
 * [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
 * [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
-* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.
+* [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Node SDK.

docs/src/integrations/voxel51.md

@@ -0,0 +1,71 @@
+![example](/assets/voxel.gif)
+
+Basic recipe
+____________
+
+The basic workflow to use LanceDB to create a similarity index on your FiftyOne
+datasets and use this to query your data is as follows:
+
+1)  Load a dataset into FiftyOne
+
+2)  Compute embedding vectors for samples or patches in your dataset, or select
+    a model to use to generate embeddings
+
+3)  Use the `compute_similarity()`
+    method to generate a LanceDB table for the samples or object
+    patches embeddings in a dataset by setting the parameter `backend="lancedb"` and
+    specifying a `brain_key` of your choice
+
+4)  Use this LanceDB table to query your data with
+    `sort_by_similarity()`
+
+5) If desired, delete the table
+
+The example below demonstrates this workflow.
+
+!!! Note
+
+    You must install the LanceDB Python client to run this
+    ```
+    pip install lancedb
+    ```
+
+```python
+
+import fiftyone as fo
+import fiftyone.brain as fob
+import fiftyone.zoo as foz
+
+# Step 1: Load your data into FiftyOne
+dataset = foz.load_zoo_dataset("quickstart")
+
+# Steps 2 and 3: Compute embeddings and create a similarity index
+lancedb_index = fob.compute_similarity(
+    dataset, 
+    model="clip-vit-base32-torch",
+    brain_key="lancedb_index",
+    backend="lancedb",
+)
+```
+Once the similarity index has been generated, we can query our data in FiftyOne
+by specifying the `brain_key`:
+
+```python
+# Step 4: Query your data
+query = dataset.first().id  # query by sample ID
+view = dataset.sort_by_similarity(
+    query, 
+    brain_key="lancedb_index",
+    k=10,  # limit to 10 most similar samples
+)
+
+# Step 5 (optional): Cleanup
+
+# Delete the LanceDB table
+lancedb_index.cleanup()
+
+# Delete run record from FiftyOne
+dataset.delete_brain_run("lancedb_index")
+```
+
+More in depth walkthrough of the integration, visit the LanceDB guide on Voxel51 - [LaceDB x Voxel51](https://docs.voxel51.com/integrations/lancedb.html)

docs/src/javascript/README.md

@@ -10,6 +10,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
@@ -28,12 +32,34 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-Run the tests with
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
 
 ```bash
 npm test
 ```
 
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript
+
+```bash
+npm run tsc
+```
+
+### Fix lints
+
 To run the linter and have it automatically fix all errors
 
 ```bash

docs/src/javascript/classes/LocalConnection.md

@@ -17,7 +17,7 @@ A connection to a LanceDB database.
 ### Properties
 
 - [\_db](LocalConnection.md#_db)
-- [\_uri](LocalConnection.md#_uri)
+- [\_options](LocalConnection.md#_options)
 
 ### Accessors
 
@@ -35,18 +35,18 @@ A connection to a LanceDB database.
 
 ### constructor
 
-• **new LocalConnection**(`db`, `uri`)
+• **new LocalConnection**(`db`, `options`)
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
 | `db` | `any` |
-| `uri` | `string` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
 
 #### Defined in
 
-[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
+[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
 
 ## Properties
 
@@ -56,17 +56,17 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
+[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
 
 ___
 
-### \_uri
+### \_options
 
-• `Private` `Readonly` **\_uri**: `string`
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
+[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
 
 ## Accessors
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
+[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
 
 ## Methods
 
@@ -112,7 +112,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
+[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
 
 ▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
@@ -134,7 +134,7 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
+[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
 
 ▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -165,7 +165,36 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
+[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
 
 ___
 
@@ -190,7 +219,7 @@ ___
 
 #### Defined in
 
-[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
+[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
 
 ___
 
@@ -216,7 +245,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
+[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
 
 ___
 
@@ -242,7 +271,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
+[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
 
 ▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -271,7 +300,34 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
+[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
 
 ___
 
@@ -291,4 +347,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)
+[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)

docs/src/javascript/classes/LocalTable.md

@@ -24,6 +24,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 - [\_embeddings](LocalTable.md#_embeddings)
 - [\_name](LocalTable.md#_name)
+- [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
 
 ### Accessors
@@ -43,7 +44,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ### constructor
 
-• **new LocalTable**<`T`\>(`tbl`, `name`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
 
 #### Type parameters
 
@@ -57,12 +58,13 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ |
 | `tbl` | `any` |
 | `name` | `string` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
 
 #### Defined in
 
-[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
+[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
 #### Type parameters
 
@@ -76,11 +78,12 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ | :------ |
 | `tbl` | `any` |  |
 | `name` | `string` |  |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
 | `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
+[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
 
 ## Properties
 
@@ -90,7 +93,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
+[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
 
 ___
 
@@ -100,7 +103,17 @@ ___
 
 #### Defined in
 
-[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
+[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
+
+___
+
+### \_options
+
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Defined in
+
+[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
 
 ___
 
@@ -110,7 +123,7 @@ ___
 
 #### Defined in
 
-[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
+[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
 
 ## Accessors
 
@@ -128,7 +141,7 @@ ___
 
 #### Defined in
 
-[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
+[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
 
 ## Methods
 
@@ -156,7 +169,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
+[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
 
 ___
 
@@ -176,7 +189,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
+[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
 
 ___
 
@@ -194,7 +207,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
 
 #### Returns
 
@@ -206,7 +219,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
 
 ___
 
@@ -232,7 +245,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
+[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
 
 ___
 
@@ -260,7 +273,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
+[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
 
 ___
 
@@ -286,4 +299,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)
+[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -62,7 +62,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
+[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
 
 ## Properties
 
@@ -72,7 +72,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
+[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
 
 ___
 
@@ -82,7 +82,7 @@ ___
 
 #### Defined in
 
-[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
+[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
 
 ___
 
@@ -92,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
+[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
 
 ___
 
@@ -102,7 +102,7 @@ ___
 
 #### Defined in
 
-[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
+[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
 
 ___
 
@@ -112,7 +112,7 @@ ___
 
 #### Defined in
 
-[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
+[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
 
 ___
 
@@ -122,7 +122,7 @@ ___
 
 #### Defined in
 
-[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
+[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
 
 ___
 
@@ -132,7 +132,7 @@ ___
 
 #### Defined in
 
-[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
+[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
 
 ___
 
@@ -142,7 +142,7 @@ ___
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
+[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
 
 ___
 
@@ -152,7 +152,7 @@ ___
 
 #### Defined in
 
-[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
+[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
 
 ___
 
@@ -162,7 +162,7 @@ ___
 
 #### Defined in
 
-[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
+[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
 
 ___
 
@@ -188,7 +188,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
+[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
 
 ## Methods
 
@@ -210,7 +210,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
+[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
 
 ___
 
@@ -232,7 +232,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
+[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
 
 ___
 
@@ -254,7 +254,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
 
 ___
 
@@ -280,7 +280,7 @@ MetricType for the different options
 
 #### Defined in
 
-[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
+[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
 
 ___
 
@@ -302,7 +302,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
+[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
 
 ___
 
@@ -324,7 +324,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
+[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
 
 ___
 
@@ -346,4 +346,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)
+[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
+[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
+[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)
+[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
+[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)
+[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / AwsCredentials
+
+# Interface: AwsCredentials
+
+## Table of contents
+
+### Properties
+
+- [accessKeyId](AwsCredentials.md#accesskeyid)
+- [secretKey](AwsCredentials.md#secretkey)
+- [sessionToken](AwsCredentials.md#sessiontoken)
+
+## Properties
+
+### accessKeyId
+
+• **accessKeyId**: `string`
+
+#### Defined in
+
+[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
+
+___
+
+### secretKey
+
+• **secretKey**: `string`
+
+#### Defined in
+
+[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
+
+___
+
+### sessionToken
+
+• `Optional` **sessionToken**: `string`
+
+#### Defined in
+
+[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/Connection.md

@@ -32,7 +32,7 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
+[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
 
 ## Methods
 
@@ -63,7 +63,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
+[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
 
 ___
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
+[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
 
 ___
 
@@ -106,7 +106,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
+[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
 
 ___
 
@@ -135,7 +135,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
+[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
 
 ___
 
@@ -149,4 +149,4 @@ ___
 
 #### Defined in
 
-[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)
+[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -0,0 +1,30 @@
+[vectordb](../README.md) / [Exports](../modules.md) / ConnectionOptions
+
+# Interface: ConnectionOptions
+
+## Table of contents
+
+### Properties
+
+- [awsCredentials](ConnectionOptions.md#awscredentials)
+- [uri](ConnectionOptions.md#uri)
+
+## Properties
+
+### awsCredentials
+
+• `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
+
+#### Defined in
+
+[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
+
+___
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -0,0 +1,149 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IvfPQIndexConfig
+
+# Interface: IvfPQIndexConfig
+
+## Table of contents
+
+### Properties
+
+- [column](IvfPQIndexConfig.md#column)
+- [index\_name](IvfPQIndexConfig.md#index_name)
+- [max\_iters](IvfPQIndexConfig.md#max_iters)
+- [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
+- [metric\_type](IvfPQIndexConfig.md#metric_type)
+- [num\_bits](IvfPQIndexConfig.md#num_bits)
+- [num\_partitions](IvfPQIndexConfig.md#num_partitions)
+- [num\_sub\_vectors](IvfPQIndexConfig.md#num_sub_vectors)
+- [replace](IvfPQIndexConfig.md#replace)
+- [type](IvfPQIndexConfig.md#type)
+- [use\_opq](IvfPQIndexConfig.md#use_opq)
+
+## Properties
+
+### column
+
+• `Optional` **column**: `string`
+
+The column to be indexed
+
+#### Defined in
+
+[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
+
+___
+
+### index\_name
+
+• `Optional` **index\_name**: `string`
+
+A unique name for the index
+
+#### Defined in
+
+[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
+
+___
+
+### max\_iters
+
+• `Optional` **max\_iters**: `number`
+
+The max number of iterations for kmeans training.
+
+#### Defined in
+
+[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
+
+___
+
+### max\_opq\_iters
+
+• `Optional` **max\_opq\_iters**: `number`
+
+Max number of iterations to train OPQ, if `use_opq` is true.
+
+#### Defined in
+
+[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
+
+___
+
+### metric\_type
+
+• `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
+
+Metric type, L2 or Cosine
+
+#### Defined in
+
+[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
+
+___
+
+### num\_bits
+
+• `Optional` **num\_bits**: `number`
+
+The number of bits to present one PQ centroid.
+
+#### Defined in
+
+[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
+
+___
+
+### num\_partitions
+
+• `Optional` **num\_partitions**: `number`
+
+The number of partitions this index
+
+#### Defined in
+
+[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
+
+___
+
+### num\_sub\_vectors
+
+• `Optional` **num\_sub\_vectors**: `number`
+
+Number of subvectors to build PQ code
+
+#### Defined in
+
+[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
+
+___
+
+### replace
+
+• `Optional` **replace**: `boolean`
+
+Replace an existing index with the same name if it exists.
+
+#### Defined in
+
+[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
+
+___
+
+### type
+
+• **type**: ``"ivf_pq"``
+
+#### Defined in
+
+[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
+
+___
+
+### use\_opq
+
+• `Optional` **use\_opq**: `boolean`
+
+Train as optimized product quantization.
+
+#### Defined in
+
+[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -52,7 +52,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
+[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
 
 ___
 
@@ -72,13 +72,13 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
+[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
 
 #### Type declaration
 
@@ -94,7 +94,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+| `indexParams` | [`IvfPQIndexConfig`](IvfPQIndexConfig.md) | The parameters of this Index, |
 
 ##### Returns
 
@@ -102,7 +102,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
+[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
 
 ___
 
@@ -116,11 +116,37 @@ ___
 
 Delete rows from this table.
 
+This can be used to delete a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [1, 2]},
+   {id: 2, vector: [3, 4]},
+   {id: 3, vector: [5, 6]},
+];
+const tbl = await con.createTable("my_table", data)
+await tbl.delete("id = 2")
+await tbl.countRows() // Returns 2
+```
+
+If you have a list of values to delete, you can combine them into a
+stringified list and use the `IN` operator:
+
+```ts
+const to_remove = [1, 5];
+await tbl.delete(`id IN (${to_remove.join(",")})`)
+await tbl.countRows() // Returns 1
+```
+
 ##### Parameters
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
 
 ##### Returns
 
@@ -128,7 +154,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
+[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
 
 ___
 
@@ -138,7 +164,7 @@ ___
 
 #### Defined in
 
-[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
+[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
 
 ___
 
@@ -166,7 +192,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
+[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
 
 ___
 
@@ -192,4 +218,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)
+[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)

docs/src/javascript/modules.md

@@ -18,8 +18,11 @@
 
 ### Interfaces
 
+- [AwsCredentials](interfaces/AwsCredentials.md)
 - [Connection](interfaces/Connection.md)
+- [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
 
 ### Type Aliases
@@ -34,11 +37,11 @@
 
 ### VectorIndexParams
 
-Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
+Ƭ **VectorIndexParams**: [`IvfPQIndexConfig`](interfaces/IvfPQIndexConfig.md)
 
 #### Defined in
 
-[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
+[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
 
 ## Functions
 
@@ -60,4 +63,20 @@ Connect to a LanceDB instance at the given URI
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)
+[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
+
+▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+
+#### Returns
+
+`Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Defined in
+
+[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)

docs/src/notebooks/code_qa_bot.ipynb

@@ -10,7 +10,11 @@
     "\n",
     "This Q&A bot will allow you to query your own documentation easily using questions. We'll also demonstrate the use of LangChain and LanceDB using the OpenAI API. \n",
     "\n",
-    "In this example we'll use Pandas 2.0 documentation, but, this could be replaced for your own docs as well"
+    "In this example we'll use Pandas 2.0 documentation, but, this could be replaced for your own docs as well\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/Code-Documentation-QA-Bot/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>\n",
+    "\n",
+    "Scripts -  [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/Code-Documentation-QA-Bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/Code-Documentation-QA-Bot/index.js)"
    ]
   },
   {
@@ -181,7 +185,7 @@
    "id": "c3852dd3",
    "metadata": {},
    "source": [
-    "# Generating emebeddings from our docs\n",
+    "# Generating embeddings from our docs\n",
     "\n",
     "Now that we have our raw documents loaded, we need to pre-process them to generate embeddings:"
    ]

docs/src/notebooks/multimodal_search.ipynb

@@ -1,5 +1,14 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "![example](https://github.com/lancedb/vectordb-recipes/assets/15766192/799f94a1-a01d-4a5b-a627-2a733bbb4227)\n",
+    "\n",
+    " <a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/multimodal_clip/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"></a>| [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/multimodal_clip/main.py)  |"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -42,6 +51,19 @@
     "## First run setup: Download data and pre-process"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Get dataset\n",
+    "\n",
+    "!wget https://eto-public.s3.us-west-2.amazonaws.com/datasets/diffusiondb_lance.tar.gz\n",
+    "!tar -xvf diffusiondb_lance.tar.gz\n",
+    "!mv diffusiondb_test rawdata.lance\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 30,
@@ -247,7 +269,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3.11.4 64-bit",
    "language": "python",
    "name": "python3"
   },
@@ -261,7 +283,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.4"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
+   }
   }
  },
  "nbformat": 4,

docs/src/notebooks/youtube_transcript_search.ipynb

@@ -8,7 +8,12 @@
    "source": [
     "# Youtube Transcript Search QA Bot\n",
     "\n",
-    "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily."
+    "This Q&A bot will allow you to search through youtube transcripts using natural language! By going through this notebook, we'll introduce how you can use LanceDB to store and manage your data easily.\n",
+    "\n",
+    "\n",
+    "<a href=\"https://colab.research.google.com/github/lancedb/vectordb-recipes/blob/main/examples/youtube_bot/main.ipynb\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\">\n",
+    "\n",
+    "Scripts - [![Python](https://img.shields.io/badge/python-3670A0?style=for-the-badge&logo=python&logoColor=ffdd54)](./examples/youtube_bot/main.py)  [![JavaScript](https://img.shields.io/badge/javascript-%23323330.svg?style=for-the-badge&logo=javascript&logoColor=%23F7DF1E)](./examples/youtube_bot/index.js)\n"
    ]
   },
   {

docs/src/python/arrow.md

@@ -5,6 +5,8 @@ Built on top of [Apache Arrow](https://arrow.apache.org/),
 `LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
 and PyArrow.
 
+## Create dataset
+
 First, we need to connect to a `LanceDB` database.
 
 ```py
@@ -27,10 +29,42 @@ data = pd.DataFrame({
 table = db.create_table("pd_table", data=data)
 ```
 
-You will find detailed instructions of creating dataset and index in
+Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
-[Basic Operations](basic.md) and [Indexing](ann_indexes.md)
+[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
+
+For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
+to lazily generate data:
+
+```py
+
+from typing import Iterable
+import pyarrow as pa
+import lancedb
+
+def make_batches() -> Iterable[pa.RecordBatch]:
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"])
+
+schema=pa.schema([
+    pa.field("vector", pa.list_(pa.float32())),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+table = db.create_table("iterable_table", data=make_batches(), schema=schema)
+```
+
+You will find detailed instructions of creating dataset in
+[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
 sections.
 
+## Vector Search
 
 We can now perform similarity search via `LanceDB` Python API.
 
@@ -45,7 +79,7 @@ print(df)
 ```
 
 ```
-    vector     item  price        score
+    vector     item  price    _distance
 0  [5.9, 26.5]  bar   20.0  14257.05957
 ```
 

docs/src/python/pydantic.md

@@ -0,0 +1,36 @@
+# Pydantic
+
+[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
+LanceDB integrates with Pydantic for schema inference, data ingestion, and query result casting.
+
+## Schema
+
+LanceDB supports to create Apache Arrow Schema from a
+[Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
+via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
+
+::: lancedb.pydantic.pydantic_to_schema
+
+## Vector Field
+
+LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
+vector Field in a Pydantic Model.
+
+::: lancedb.pydantic.vector
+
+## Type Conversion
+
+LanceDB automatically convert Pydantic fields to
+[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
+
+Current supported type conversions:
+
+| Pydantic Field Type | PyArrow Data Type |
+| ------------------- | ----------------- |
+| `int`               | `pyarrow.int64`   |
+| `float`              | `pyarrow.float64`  |
+| `bool`              | `pyarrow.bool`    |
+| `str`               | `pyarrow.utf8()`    |
+| `list`              | `pyarrow.List`    |
+| `BaseModel`         | `pyarrow.Struct`    |
+| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -46,10 +46,6 @@ pip install lancedb
 
 ## Utilities
 
-::: lancedb.schema.schema_to_dict
-
-::: lancedb.schema.dict_to_schema
-
 ::: lancedb.vector
 
 ## Integrations
@@ -60,4 +56,4 @@ pip install lancedb
 
 ::: lancedb.pydantic.vector
 
-
+::: lancedb.pydantic.LanceModel

docs/test/md_testing.js

@@ -5,9 +5,12 @@ const path = require("path");
 const excludedFiles = [
   "../src/fts.md",
   "../src/embedding.md",
+  "../src/ann_indexes.md",
   "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
   "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/transformerjs_embedding_search_nodejs.md",
   "../src/examples/youtube_transcript_bot_with_nodejs.md",
+  "../src/guides/tables.md",
 ];
 const nodePrefix = "javascript";
 const nodeFile = ".js";
@@ -48,4 +51,4 @@ for (const file of files.filter((file) => !excludedFiles.includes(file))) {
     fs.mkdirSync(path.dirname(outPath), { recursive: true });
     fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
   }
-}
+}

docs/test/md_testing.py

@@ -7,7 +7,9 @@ excluded_files = [
     "../src/embedding.md",
     "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
     "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
-    "../src/examples/youtube_transcript_bot_with_nodejs.md"
+    "../src/examples/youtube_transcript_bot_with_nodejs.md",
+    "../src/integrations/voxel51.md",
+    "../src/guides/tables.md"
 ]
 
 python_prefix = "py"

node/examples/js-transformers/index.js

@@ -0,0 +1,66 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+
+async function example() {
+
+    const lancedb = require('vectordb')
+
+    // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
+    const { pipeline } = await import('@xenova/transformers')
+    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+
+
+    // Create embedding function from pipeline which returns a list of vectors from batch
+    // sourceColumn is the name of the column in the data to be embedded
+    //
+    // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
+    const embed_fun = {}
+    embed_fun.sourceColumn = 'text'
+    embed_fun.embed = async function (batch) {
+        let result = []
+        for (let text of batch) {
+            const res = await pipe(text, { pooling: 'mean', normalize: true })
+            result.push(Array.from(res['data']))
+        }
+        return (result)
+    }
+
+    // Link a folder and create a table with data
+    const db = await lancedb.connect('data/sample-lancedb')
+
+    const data = [
+        { id: 1, text: 'Cherry', type: 'fruit' },
+        { id: 2, text: 'Carrot', type: 'vegetable' },
+        { id: 3, text: 'Potato', type: 'vegetable' },
+        { id: 4, text: 'Apple', type: 'fruit' },
+        { id: 5, text: 'Banana', type: 'fruit' }
+    ]
+
+    const table = await db.createTable('food_table', data, embed_fun)
+
+
+    // Query the table
+    const results = await table
+        .search("a sweet fruit to eat")
+        .metricType("cosine")
+        .limit(2)
+        .execute()
+    console.log(results.map(r => r.text))
+
+}
+
+example().then(_ => { console.log("Done!") })

node/examples/js-transformers/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "vectordb-example-js-transformers",
+  "version": "1.0.0",
+  "description": "Example for using transformers.js with lancedb",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@xenova/transformers": "^2.4.1",
+    "vectordb": "file:../.."
+  }
+
+}

node/native.js

@@ -12,26 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const { currentTarget } = require('@neon-rs/load');
+const { currentTarget } = require('@neon-rs/load')
 
-let nativeLib;
+let nativeLib
 
 try {
-    nativeLib = require(`vectordb-${currentTarget()}`);
+  // When developing locally, give preference to the local built library
-} catch (e) {
+  nativeLib = require('./index.node')
-    try {
+} catch {
-        // Might be developing locally, so try that. But don't expose that error
+  try {
-        // to the user.
+    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`)
-        nativeLib = require("./index.node");
+  } catch (e) {
-    } catch {
+    throw new Error(`vectordb: failed to load native library.
-        throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
-  You may need to run \`npm install vectordb-${currentTarget()}\`.
+
-      
   If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
       
-  Source error: ${e}`);
+  Source error: ${e}`)
-    }   
+  }
 }
 
 // Dynamic require for runtime.
-module.exports = nativeLib;
+module.exports = nativeLib

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.1.12",
+  "version": "0.1.19",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.12",
+      "version": "0.1.19",
       "cpu": [
         "x64",
         "arm64"
@@ -14,15 +14,17 @@
       "license": "Apache-2.0",
       "os": [
         "darwin",
-        "linux"
+        "linux",
+        "win32"
       ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
         "@neon-rs/load": "^0.0.74",
-        "apache-arrow": "^12.0.0"
+        "apache-arrow": "^12.0.0",
+        "axios": "^1.4.0"
       },
       "devDependencies": {
-        "@neon-rs/cli": "^0.0.74",
+        "@neon-rs/cli": "^0.0.160",
         "@types/chai": "^4.3.4",
         "@types/chai-as-promised": "^7.1.5",
         "@types/mocha": "^10.0.1",
@@ -49,10 +51,11 @@
         "typescript": "*"
       },
       "optionalDependencies": {
-        "vectordb-darwin-arm64": "0.1.12",
+        "@lancedb/vectordb-darwin-arm64": "0.1.19",
-        "vectordb-darwin-x64": "0.1.12",
+        "@lancedb/vectordb-darwin-x64": "0.1.19",
-        "vectordb-linux-arm64-gnu": "0.1.12",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-        "vectordb-linux-x64-gnu": "0.1.12"
+        "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
+        "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -82,6 +85,97 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
     },
+    "node_modules/@cargo-messages/android-arm-eabi": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
+      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "android"
+      ]
+    },
+    "node_modules/@cargo-messages/darwin-arm64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
+      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@cargo-messages/darwin-x64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
+      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@cargo-messages/linux-arm-gnueabihf": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
+      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@cargo-messages/linux-x64-gnu": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
+      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@cargo-messages/win32-arm64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
+      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/@cargo-messages/win32-x64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
+      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -220,13 +314,82 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
+      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-darwin-x64": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
+      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
+      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
+      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
+      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
     "node_modules/@neon-rs/cli": {
-      "version": "0.0.74",
+      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
       "dev": true,
       "bin": {
         "neon": "index.js"
+      },
+      "optionalDependencies": {
+        "@cargo-messages/android-arm-eabi": "0.0.160",
+        "@cargo-messages/darwin-arm64": "0.0.160",
+        "@cargo-messages/darwin-x64": "0.0.160",
+        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
+        "@cargo-messages/linux-x64-gnu": "0.0.160",
+        "@cargo-messages/win32-arm64-msvc": "0.0.160",
+        "@cargo-messages/win32-x64-msvc": "0.0.160"
       }
     },
     "node_modules/@neon-rs/load": {
@@ -840,8 +1003,7 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-      "dev": true
     },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
@@ -856,12 +1018,13 @@
       }
     },
     "node_modules/axios": {
-      "version": "0.26.1",
+      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
-      "dev": true,
       "dependencies": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "node_modules/balanced-match": {
@@ -1092,7 +1255,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -1315,7 +1477,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2082,7 +2243,6 @@
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
       "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -2111,7 +2271,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -2985,7 +3144,6 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -2994,7 +3152,6 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -3288,6 +3445,15 @@
         "form-data": "^4.0.0"
       }
     },
+    "node_modules/openai/node_modules/axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
     "node_modules/optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -3439,6 +3605,11 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "node_modules/punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -4531,6 +4702,55 @@
         }
       }
     },
+    "@cargo-messages/android-arm-eabi": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/android-arm-eabi/-/android-arm-eabi-0.0.160.tgz",
+      "integrity": "sha512-PTgCEmBHEPKJbxwlHVXB3aGES+NqpeBvn6hJNYWIkET3ZQCSJnScMlIDQXEkWndK7J+hW3Or3H32a93B/MbbfQ==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/darwin-arm64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-arm64/-/darwin-arm64-0.0.160.tgz",
+      "integrity": "sha512-YSVUuc8TUTi/XmZVg9KrH0bDywKLqC1zeTyZYAYDDmqVDZW9KeTnbBUECKRs56iyHeO+kuEkVW7MKf7j2zb/FA==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/darwin-x64": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/darwin-x64/-/darwin-x64-0.0.160.tgz",
+      "integrity": "sha512-U+YlAR+9tKpBljnNPWMop5YhvtwfIPQSAaUYN2llteC7ZNU5/cv8CGT1vm7uFNxr2LeGuAtRbzIh2gUmTV8mng==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/linux-arm-gnueabihf": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-arm-gnueabihf/-/linux-arm-gnueabihf-0.0.160.tgz",
+      "integrity": "sha512-wqAelTzVv1E7Ls4aviqUbem5xjzCaJQxQtVnLhv6pf1k0UyEHCS2WdufFFmWcojGe7QglI4uve3KTe01MKYj0A==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/linux-x64-gnu": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/linux-x64-gnu/-/linux-x64-gnu-0.0.160.tgz",
+      "integrity": "sha512-LQ6e7O7YYkWfDNIi/53q2QG/+lZok72LOG+NKDVCrrY4TYUcrTqWAybOV6IlkVntKPnpx8YB95umSQGeVuvhpQ==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/win32-arm64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-arm64-msvc/-/win32-arm64-msvc-0.0.160.tgz",
+      "integrity": "sha512-VDMBhyun02gIDwmEhkYP1W9Z0tYqn4drgY5Iua1qV2tYOU58RVkWhzUYxM9rzYbnwKZlltgM46J/j5QZ3VaFrA==",
+      "dev": true,
+      "optional": true
+    },
+    "@cargo-messages/win32-x64-msvc": {
+      "version": "0.0.160",
+      "resolved": "https://registry.npmjs.org/@cargo-messages/win32-x64-msvc/-/win32-x64-msvc-0.0.160.tgz",
+      "integrity": "sha512-vnoglDxF6zj0W/Co9D0H/bgnrhUuO5EumIf9v3ujLtBH94rAX11JsXh/FgC/8wQnQSsLyWSq70YxNS2wdETxjA==",
+      "dev": true,
+      "optional": true
+    },
     "@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -4631,11 +4851,50 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@lancedb/vectordb-darwin-arm64": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.19.tgz",
+      "integrity": "sha512-efQhJkBKvMNhjFq3Sw3/qHo9D9gb9UqiIr98n3STsbNxBQjMnWemXn91Ckl40siRG1O8qXcINW7Qs/EGmus+kg==",
+      "optional": true
+    },
+    "@lancedb/vectordb-darwin-x64": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.19.tgz",
+      "integrity": "sha512-r6OZNVyemAssABz2w7CRhe7dyREwBEfTytn+ux1zzTnzsgMgDovCQ0rQ3WZcxWvcy7SFCxiemA9IP1b/lsb4tQ==",
+      "optional": true
+    },
+    "@lancedb/vectordb-linux-arm64-gnu": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.1.19.tgz",
+      "integrity": "sha512-mL/hRmZp6Kw7hmGJBdOZfp/tTYiCdlOcs8DA/+nr2eiXERv0gIhyiKvr2P5DwbBmut3qXEkDalMHTo95BSdL2A==",
+      "optional": true
+    },
+    "@lancedb/vectordb-linux-x64-gnu": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.19.tgz",
+      "integrity": "sha512-AG0FHksbbr+cHVKPi4B8cmBtqb6T9E0uaK4kyZkXrX52/xtv9RYVZcykaB/tSSm0XNFPWWRnx9R8UqNZV/hxMA==",
+      "optional": true
+    },
+    "@lancedb/vectordb-win32-x64-msvc": {
+      "version": "0.1.19",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.1.19.tgz",
+      "integrity": "sha512-PDWZ2hvLVXH4Z4WIO1rsWY8ev3NpNm7aXlaey32P+l1Iz9Hia9+F2GBpp2UiEQKfvbk82ucAvBLRmpSsHY8Tlw==",
+      "optional": true
+    },
     "@neon-rs/cli": {
-      "version": "0.0.74",
+      "version": "0.0.160",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "integrity": "sha512-GQjzHPJVTOARbX3nP/fAWqBq7JlQ8XgfYlCa+iwzIXf0LC1EyfJTX+vqGD/36b9lKoyY01Z/aDUB9o/qF6ztHA==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "@cargo-messages/android-arm-eabi": "0.0.160",
+        "@cargo-messages/darwin-arm64": "0.0.160",
+        "@cargo-messages/darwin-x64": "0.0.160",
+        "@cargo-messages/linux-arm-gnueabihf": "0.0.160",
+        "@cargo-messages/linux-x64-gnu": "0.0.160",
+        "@cargo-messages/win32-arm64-msvc": "0.0.160",
+        "@cargo-messages/win32-x64-msvc": "0.0.160"
+      }
     },
     "@neon-rs/load": {
       "version": "0.0.74",
@@ -5097,8 +5356,7 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
-      "dev": true
     },
     "available-typed-arrays": {
       "version": "1.0.5",
@@ -5107,12 +5365,13 @@
       "dev": true
     },
     "axios": {
-      "version": "0.26.1",
+      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
-      "dev": true,
       "requires": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "balanced-match": {
@@ -5292,7 +5551,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
       }
@@ -5459,8 +5717,7 @@
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
-      "dev": true
     },
     "diff": {
       "version": "4.0.2",
@@ -6030,8 +6287,7 @@
     "follow-redirects": {
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
-      "dev": true
     },
     "for-each": {
       "version": "0.3.3",
@@ -6046,7 +6302,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -6660,14 +6915,12 @@
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
-      "dev": true
     },
     "mime-types": {
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "requires": {
         "mime-db": "1.52.0"
       }
@@ -6893,6 +7146,17 @@
       "requires": {
         "axios": "^0.26.0",
         "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.26.1",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+          "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+          "dev": true,
+          "requires": {
+            "follow-redirects": "^1.14.8"
+          }
+        }
       }
     },
     "optionator": {
@@ -7001,6 +7265,11 @@
       "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
       "dev": true
     },
+    "proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.1.13",
+  "version": "0.1.19",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,8 +8,8 @@
     "tsc": "tsc -b",
     "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
-    "test": "npm run tsc; mocha -recursive dist/test",
+    "test": "npm run tsc && mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts",
+    "lint": "eslint native.js src --ext .js,.ts",
     "clean": "rm -rf node_modules *.node dist/",
     "pack-build": "neon pack-build",
     "check-npm": "printenv && which node && which npm && npm --version"
@@ -27,7 +27,7 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@neon-rs/cli": "^0.0.74",
+    "@neon-rs/cli": "^0.0.160",
     "@types/chai": "^4.3.4",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
@@ -56,11 +56,13 @@
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^12.0.0"
+    "apache-arrow": "^12.0.0",
+    "axios": "^1.4.0"
   },
   "os": [
     "darwin",
-    "linux"
+    "linux",
+    "win32"
   ],
   "cpu": [
     "x64",
@@ -68,16 +70,18 @@
   ],
   "neon": {
     "targets": {
-      "x86_64-apple-darwin": "vectordb-darwin-x64",
+      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
-      "aarch64-apple-darwin": "vectordb-darwin-arm64",
+      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
+      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu"
+      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu",
+      "x86_64-pc-windows-msvc": "@lancedb/vectordb-win32-x64-msvc"
     }
   },
   "optionalDependencies": {
-    "vectordb-darwin-arm64": "0.1.13",
+    "@lancedb/vectordb-darwin-arm64": "0.1.19",
-    "vectordb-darwin-x64": "0.1.13",
+    "@lancedb/vectordb-darwin-x64": "0.1.19",
-    "vectordb-linux-x64-gnu": "0.1.13",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.1.19",
-    "vectordb-linux-arm64-gnu": "0.1.13"
+    "@lancedb/vectordb-linux-x64-gnu": "0.1.19",
+    "@lancedb/vectordb-win32-x64-msvc": "0.1.19"
   }
 }

node/src/embedding/embedding_function.ts

@@ -26,3 +26,8 @@ export interface EmbeddingFunction<T> {
      */
   embed: (data: T[]) => Promise<number[][]>
 }
+
+export function isEmbeddingFunction<T> (value: any): value is EmbeddingFunction<T> {
+  return typeof value.sourceColumn === 'string' &&
+      typeof value.embed === 'function'
+}

node/src/index.ts

@@ -14,16 +14,18 @@
 
 import {
   RecordBatchFileWriter,
-  type Table as ArrowTable,
+  type Table as ArrowTable
-  tableFromIPC,
-  Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
+import { RemoteConnection } from './remote'
+import { Query } from './query'
+import { isEmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
 
+export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
@@ -37,7 +39,16 @@ export interface AwsCredentials {
 
 export interface ConnectionOptions {
   uri: string
+
   awsCredentials?: AwsCredentials
+
+  // API key for the remote connections
+  apiKey?: string
+  // Region to connect
+  region?: string
+
+  // override the host for the remote connections
+  hostOverride?: string
 }
 
 /**
@@ -54,9 +65,16 @@ export async function connect (arg: string | Partial<ConnectionOptions>): Promis
     // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
     opts = Object.assign({
       uri: '',
-      awsCredentials: undefined
+      awsCredentials: undefined,
+      apiKey: undefined,
+      region: 'us-west-2'
     }, arg)
   }
+
+  if (opts.uri.startsWith('db://')) {
+    // Remote connection
+    return new RemoteConnection(opts)
+  }
   const db = await databaseNew(opts.uri)
   return new LocalConnection(db, opts)
 }
@@ -84,10 +102,35 @@ export interface Connection {
    *
    * @param {string} name - The name of the table.
    * @param data - Non-empty Array of Records to be inserted into the table
-   * @param {WriteMode} mode - The write mode to use when creating the table.
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
    */
-  createTable<T>(name: string, data: Array<Record<string, unknown>>, mode?: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
 
   createTableArrow(name: string, table: ArrowTable): Promise<Table>
 
@@ -142,7 +185,34 @@ export interface Table<T = number[]> {
   /**
    * Delete rows from this table.
    *
-   * @param filter  A filter in the same format used by a sql WHERE clause.
+   * This can be used to delete a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause. The
+   *                filter must not be empty.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [1, 2]},
+   *    {id: 2, vector: [3, 4]},
+   *    {id: 3, vector: [5, 6]},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   * await tbl.delete("id = 2")
+   * await tbl.countRows() // Returns 2
+   * ```
+   *
+   * If you have a list of values to delete, you can combine them into a
+   * stringified list and use the `IN` operator:
+   *
+   * ```ts
+   * const to_remove = [1, 5];
+   * await tbl.delete(`id IN (${to_remove.join(",")})`)
+   * await tbl.countRows() // Returns 1
+   * ```
    */
   delete: (filter: string) => Promise<void>
 }
@@ -164,8 +234,8 @@ export class LocalConnection implements Connection {
   }
 
   /**
-     * Get the names of all tables in the database.
+   * Get the names of all tables in the database.
-     */
+   */
   async tableNames (): Promise<string[]> {
     return databaseTableNames.call(this._db)
   }
@@ -176,6 +246,7 @@ export class LocalConnection implements Connection {
    * @param name The name of the table.
    */
   async openTable (name: string): Promise<Table>
+
   /**
    * Open a table in the database.
    *
@@ -193,32 +264,19 @@ export class LocalConnection implements Connection {
     }
   }
 
-  /**
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, optsOrEmbedding?: WriteOptions | EmbeddingFunction<T>, opt?: WriteOptions): Promise<Table<T>> {
-   * Creates a new Table and initialize it with new data.
+    let writeOptions: WriteOptions = new DefaultWriteOptions()
-   *
+    if (opt !== undefined && isWriteOptions(opt)) {
-   * @param name The name of the table.
+      writeOptions = opt
-   * @param data Non-empty Array of Records to be inserted into the Table
+    } else if (optsOrEmbedding !== undefined && isWriteOptions(optsOrEmbedding)) {
-   * @param mode The write mode to use when creating the table.
+      writeOptions = optsOrEmbedding
-   */
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode?: WriteMode): Promise<Table>
-  async createTable (name: string, data: Array<Record<string, unknown>>, mode: WriteMode): Promise<Table>
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param mode The write mode to use when creating the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    if (mode === undefined) {
-      mode = WriteMode.Create
     }
 
-    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase()]
+    let embeddings: undefined | EmbeddingFunction<T>
+    if (optsOrEmbedding !== undefined && isEmbeddingFunction(optsOrEmbedding)) {
+      embeddings = optsOrEmbedding
+    }
+    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), writeOptions.writeMode?.toString()]
     if (this._options.awsCredentials !== undefined) {
       createArgs.push(this._options.awsCredentials.accessKeyId)
       createArgs.push(this._options.awsCredentials.secretKey)
@@ -252,7 +310,7 @@ export class LocalConnection implements Connection {
 }
 
 export class LocalTable<T = number[]> implements Table<T> {
-  private readonly _tbl: any
+  private _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
   private readonly _options: ConnectionOptions
@@ -281,7 +339,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param query The query search term
    */
   search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+    return new Query(query, this._tbl, this._embeddings)
   }
 
   /**
@@ -299,7 +357,7 @@ export class LocalTable<T = number[]> implements Table<T> {
         callArgs.push(this._options.awsCredentials.sessionToken)
       }
     }
-    return tableAdd.call(...callArgs)
+    return tableAdd.call(...callArgs).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -317,7 +375,7 @@ export class LocalTable<T = number[]> implements Table<T> {
         callArgs.push(this._options.awsCredentials.sessionToken)
       }
     }
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
+    return tableAdd.call(...callArgs).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -326,7 +384,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param indexParams The parameters of this Index, @see VectorIndexParams.
    */
   async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    return tableCreateVectorIndex.call(this._tbl, indexParams)
+    return tableCreateVectorIndex.call(this._tbl, indexParams).then((newTable: any) => { this._tbl = newTable })
   }
 
   /**
@@ -342,7 +400,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param filter A filter in the same format used by a sql WHERE clause.
    */
   async delete (filter: string): Promise<void> {
-    return tableDelete.call(this._tbl, filter)
+    return tableDelete.call(this._tbl, filter).then((newTable: any) => { this._tbl = newTable })
   }
 }
 
@@ -403,116 +461,6 @@ export interface IvfPQIndexConfig {
 
 export type VectorIndexParams = IvfPQIndexConfig
 
-/**
- * A builder for nearest neighbor queries for LanceDB.
- */
-export class Query<T = number[]> {
-  private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
-  private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
-  private _filter?: string
-  private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
-
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
-    this._tbl = tbl
-    this._query = query
-    this._limit = 10
-    this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
-    this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
-  }
-
-  /***
-   * Sets the number of results that will be returned
-   * @param value number of results
-   */
-  limit (value: number): Query<T> {
-    this._limit = value
-    return this
-  }
-
-  /**
-   * Refine the results by reading extra elements and re-ranking them in memory.
-   * @param value refine factor to use in this query.
-   */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-   * The number of probes used. A higher number makes search more accurate but also slower.
-   * @param value The number of probes used.
-   */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-   * A filter statement to be applied to this query.
-   * @param value A filter in the same format used by a sql WHERE clause.
-   */
-  filter (value: string): Query<T> {
-    this._filter = value
-    return this
-  }
-
-  where = this.filter
-
-  /** Return only the specified columns.
-   *
-   * @param value Only select the specified columns. If not specified, all columns will be returned.
-   */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
-  /**
-   * The MetricType used for this Query.
-   * @param value The metric to the. @see MetricType for the different options
-   */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-   * Execute the query and return the results as an Array of Objects
-   */
-  async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
-    const buffer = await tableSearch.call(this._tbl, this)
-    const data = tableFromIPC(buffer)
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-}
-
 /**
  * Write mode for writing a table.
  */
@@ -525,6 +473,23 @@ export enum WriteMode {
   Append = 'append'
 }
 
+/**
+ * Write options when creating a Table.
+ */
+export interface WriteOptions {
+  /** A {@link WriteMode} to use on this operation */
+  writeMode?: WriteMode
+}
+
+export class DefaultWriteOptions implements WriteOptions {
+  writeMode = WriteMode.Create
+}
+
+export function isWriteOptions (value: any): value is WriteOptions {
+  return Object.keys(value).length === 1 &&
+      (value.writeMode === undefined || typeof value.writeMode === 'string')
+}
+
 /**
  * Distance metrics type.
  */

node/src/query.ts

@@ -0,0 +1,130 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Vector, tableFromIPC } from 'apache-arrow'
+import { type EmbeddingFunction } from './embedding/embedding_function'
+import { type MetricType } from '.'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { tableSearch } = require('../native.js')
+
+/**
+ * A builder for nearest neighbor queries for LanceDB.
+ */
+export class Query<T = number[]> {
+  private readonly _query: T
+  private readonly _tbl?: any
+  private _queryVector?: number[]
+  private _limit: number
+  private _refineFactor?: number
+  private _nprobes: number
+  private _select?: string[]
+  private _filter?: string
+  private _metricType?: MetricType
+  protected readonly _embeddings?: EmbeddingFunction<T>
+
+  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+    this._tbl = tbl
+    this._query = query
+    this._limit = 10
+    this._nprobes = 20
+    this._refineFactor = undefined
+    this._select = undefined
+    this._filter = undefined
+    this._metricType = undefined
+    this._embeddings = embeddings
+  }
+
+  /***
+     * Sets the number of results that will be returned
+     * @param value number of results
+     */
+  limit (value: number): Query<T> {
+    this._limit = value
+    return this
+  }
+
+  /**
+     * Refine the results by reading extra elements and re-ranking them in memory.
+     * @param value refine factor to use in this query.
+     */
+  refineFactor (value: number): Query<T> {
+    this._refineFactor = value
+    return this
+  }
+
+  /**
+     * The number of probes used. A higher number makes search more accurate but also slower.
+     * @param value The number of probes used.
+     */
+  nprobes (value: number): Query<T> {
+    this._nprobes = value
+    return this
+  }
+
+  /**
+     * A filter statement to be applied to this query.
+     * @param value A filter in the same format used by a sql WHERE clause.
+     */
+  filter (value: string): Query<T> {
+    this._filter = value
+    return this
+  }
+
+  where = this.filter
+
+  /** Return only the specified columns.
+     *
+     * @param value Only select the specified columns. If not specified, all columns will be returned.
+     */
+  select (value: string[]): Query<T> {
+    this._select = value
+    return this
+  }
+
+  /**
+     * The MetricType used for this Query.
+     * @param value The metric to the. @see MetricType for the different options
+     */
+  metricType (value: MetricType): Query<T> {
+    this._metricType = value
+    return this
+  }
+
+  /**
+     * Execute the query and return the results as an Array of Objects
+     */
+  async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    if (this._embeddings !== undefined) {
+      this._queryVector = (await this._embeddings.embed([this._query]))[0]
+    } else {
+      this._queryVector = this._query as number[]
+    }
+
+    const buffer = await tableSearch.call(this._tbl, this)
+    const data = tableFromIPC(buffer)
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}

node/src/remote/client.ts

@@ -0,0 +1,137 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios, { type AxiosResponse } from 'axios'
+
+import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
+
+export class HttpLancedbClient {
+  private readonly _url: string
+  private readonly _apiKey: () => string
+
+  public constructor (
+    url: string,
+    apiKey: string,
+    private readonly _dbName?: string
+  ) {
+    this._url = url
+    this._apiKey = () => apiKey
+  }
+
+  get uri (): string {
+    return this._url
+  }
+
+  public async search (
+    tableName: string,
+    vector: number[],
+    k: number,
+    nprobes: number,
+    refineFactor?: number,
+    columns?: string[],
+    filter?: string
+  ): Promise<ArrowTable<any>> {
+    const response = await axios.post(
+              `${this._url}/v1/table/${tableName}/query/`,
+              {
+                vector,
+                k,
+                nprobes,
+                refineFactor,
+                columns,
+                filter
+              },
+              {
+                headers: {
+                  'Content-Type': 'application/json',
+                  'x-api-key': this._apiKey(),
+                  ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+                },
+                responseType: 'arraybuffer',
+                timeout: 10000
+              }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+
+    const table = tableFromIPC(response.data)
+    return table
+  }
+
+  /**
+   * Sent GET request.
+   */
+  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
+    const response = await axios.get(
+      `${this._url}${path}`,
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this._apiKey()
+        },
+        params,
+        timeout: 10000
+      }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+    return response
+  }
+
+  /**
+   * Sent POST request.
+   */
+  public async post (path: string, data?: any, params?: Record<string, string | number>): Promise<AxiosResponse> {
+    const response = await axios.post(
+        `${this._url}${path}`,
+        data,
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            'x-api-key': this._apiKey(),
+            ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
+          },
+          params,
+          timeout: 30000
+        }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+          `Server Error, status: ${response.status as number}, ` +
+          `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+    return response
+  }
+}

node/src/remote/index.ts

@@ -0,0 +1,168 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
+  type ConnectionOptions
+} from '../index'
+import { Query } from '../query'
+
+import { type Table as ArrowTable, Vector } from 'apache-arrow'
+import { HttpLancedbClient } from './client'
+
+/**
+ * Remote connection.
+ */
+export class RemoteConnection implements Connection {
+  private readonly _client: HttpLancedbClient
+  private readonly _dbName: string
+
+  constructor (opts: ConnectionOptions) {
+    if (!opts.uri.startsWith('db://')) {
+      throw new Error(`Invalid remote DB URI: ${opts.uri}`)
+    }
+    if (opts.apiKey === undefined || opts.region === undefined) {
+      throw new Error('API key and region are not supported for remote connections')
+    }
+
+    this._dbName = opts.uri.slice('db://'.length)
+    let server: string
+    if (opts.hostOverride === undefined) {
+      server = `https://${this._dbName}.${opts.region}.api.lancedb.com`
+    } else {
+      server = opts.hostOverride
+    }
+    this._client = new HttpLancedbClient(server, opts.apiKey, opts.hostOverride === undefined ? undefined : this._dbName)
+  }
+
+  get uri (): string {
+    // add the lancedb+ prefix back
+    return 'db://' + this._client.uri
+  }
+
+  async tableNames (): Promise<string[]> {
+    const response = await this._client.get('/v1/table/')
+    return response.data.tables
+  }
+
+  async openTable (name: string): Promise<Table>
+  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (embeddings !== undefined) {
+      return new RemoteTable(this._client, name, embeddings)
+    } else {
+      return new RemoteTable(this._client, name)
+    }
+  }
+
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    throw new Error('Not implemented')
+  }
+
+  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
+    throw new Error('Not implemented')
+  }
+
+  async dropTable (name: string): Promise<void> {
+    await this._client.post(`/v1/table/${name}/drop/`)
+  }
+}
+
+export class RemoteQuery<T = number[]> extends Query<T> {
+  constructor (query: T, private readonly _client: HttpLancedbClient,
+    private readonly _name: string, embeddings?: EmbeddingFunction<T>) {
+    super(query, undefined, embeddings)
+  }
+
+  // TODO: refactor this to a base class + queryImpl pattern
+  async execute<T = Record<string, unknown>>(): Promise<T[]> {
+    const embeddings = this._embeddings
+    const query = (this as any)._query
+    let queryVector: number[]
+
+    if (embeddings !== undefined) {
+      queryVector = (await embeddings.embed([query]))[0]
+    } else {
+      queryVector = query as number[]
+    }
+
+    const data = await this._client.search(
+      this._name,
+      queryVector,
+      (this as any)._limit,
+      (this as any)._nprobes,
+      (this as any)._refineFactor,
+      (this as any)._select,
+      (this as any)._filter
+    )
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}
+
+// we are using extend until we have next next version release
+// Table and Connection has both been refactored to interfaces
+export class RemoteTable<T = number[]> implements Table<T> {
+  private readonly _client: HttpLancedbClient
+  private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _name: string
+
+  constructor (client: HttpLancedbClient, name: string)
+  constructor (client: HttpLancedbClient, name: string, embeddings: EmbeddingFunction<T>)
+  constructor (client: HttpLancedbClient, name: string, embeddings?: EmbeddingFunction<T>) {
+    this._client = client
+    this._name = name
+    this._embeddings = embeddings
+  }
+
+  get name (): string {
+    return this._name
+  }
+
+  search (query: T): Query<T> {
+    return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
+  }
+
+  async add (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+    throw new Error('Not implemented')
+  }
+
+  async countRows (): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async delete (filter: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+}

node/src/test/embedding/openai.ts

@@ -16,6 +16,7 @@ import { describe } from 'mocha'
 import { assert } from 'chai'
 
 import { OpenAIEmbeddingFunction } from '../../embedding/openai'
+import { isEmbeddingFunction } from '../../embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { OpenAIApi } = require('openai')
@@ -47,4 +48,10 @@ describe('OpenAPIEmbeddings', function () {
       assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
     })
   })
+
+  describe('isEmbeddingFunction', function () {
+    it('should match the isEmbeddingFunction guard', function () {
+      assert.isTrue(isEmbeddingFunction(new OpenAIEmbeddingFunction('text', 'sk-key')))
+    })
+  })
 })

node/src/test/test.ts

@@ -18,7 +18,7 @@ import * as chai from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions } from '../index'
 
 const expect = chai.expect
 const assert = chai.assert
@@ -107,9 +107,9 @@ describe('LanceDB client', function () {
       const table = await con.openTable('vectors')
       const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
       assert.equal(results.length, 2)
-      // vector and score are always returned
+      // vector and _distance are always returned
       assert.isDefined(results[0].vector)
-      assert.isDefined(results[0].score)
+      assert.isDefined(results[0]._distance)
       assert.isDefined(results[0].is_active)
 
       assert.isUndefined(results[0].id)
@@ -134,6 +134,18 @@ describe('LanceDB client', function () {
       assert.equal(await table.countRows(), 2)
     })
 
+    it('fails to create a new table when the vector column is missing', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, price: 10 }
+      ]
+
+      const create = con.createTable('missing_vector', data)
+      await expect(create).to.be.rejectedWith(Error, 'column \'vector\' is missing')
+    })
+
     it('use overwrite flag to overwrite existing table', async function () {
       const dir = await track().mkdir('lancejs')
       const con = await lancedb.connect(dir)
@@ -144,7 +156,7 @@ describe('LanceDB client', function () {
       ]
 
       const tableName = 'overwrite'
-      await con.createTable(tableName, data, WriteMode.Create)
+      await con.createTable(tableName, data, { writeMode: WriteMode.Create })
 
       const newData = [
         { id: 1, vector: [0.1, 0.2], price: 10 },
@@ -154,7 +166,7 @@ describe('LanceDB client', function () {
 
       await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
 
-      const table = await con.createTable(tableName, newData, WriteMode.Overwrite)
+      const table = await con.createTable(tableName, newData, { writeMode: WriteMode.Overwrite })
       assert.equal(table.name, tableName)
       assert.equal(await table.countRows(), 3)
     })
@@ -230,6 +242,22 @@ describe('LanceDB client', function () {
       // Default replace = true
       await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
     }).timeout(50_000)
+
+    it('it should fail when the column is not a vector', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      await expect(createIndex).to.be.rejectedWith(/VectorIndex requires the column data type to be fixed size list of float32s/)
+    })
+
+    it('it should fail when the column is not a vector', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
+      await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
+    })
   })
 
   describe('when using a custom embedding function', function () {
@@ -259,7 +287,7 @@ describe('LanceDB client', function () {
         { price: 10, name: 'foo' },
         { price: 50, name: 'bar' }
       ]
-      const table = await con.createTable('vectors', data, WriteMode.Create, embeddings)
+      const table = await con.createTable('vectors', data, embeddings, { writeMode: WriteMode.Create })
       const results = await table.search('foo').execute()
       assert.equal(results.length, 2)
     })
@@ -268,7 +296,7 @@ describe('LanceDB client', function () {
 
 describe('Query object', function () {
   it('sets custom parameters', async function () {
-    const query = new Query(undefined, [0.1, 0.3])
+    const query = new Query([0.1, 0.3])
       .limit(1)
       .metricType(MetricType.Cosine)
       .refineFactor(100)
@@ -317,3 +345,20 @@ describe('Drop table', function () {
     assert.deepEqual(await con.tableNames(), ['t2'])
   })
 })
+
+describe('WriteOptions', function () {
+  context('#isWriteOptions', function () {
+    it('should not match empty object', function () {
+      assert.equal(isWriteOptions({}), false)
+    })
+    it('should match write options', function () {
+      assert.equal(isWriteOptions({ writeMode: WriteMode.Create }), true)
+    })
+    it('should match undefined write mode', function () {
+      assert.equal(isWriteOptions({ writeMode: undefined }), true)
+    })
+    it('should match default write options', function () {
+      assert.equal(isWriteOptions(new DefaultWriteOptions()), true)
+    })
+  })
+})

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.8
+current_version = 0.2.0
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/__init__.py

@@ -19,7 +19,11 @@ from .schema import vector
 
 
 def connect(
-    uri: URI, *, api_key: Optional[str] = None, region: str = "us-west-2"
+    uri: URI,
+    *,
+    api_key: Optional[str] = None,
+    region: str = "us-west-2",
+    host_override: Optional[str] = None,
 ) -> DBConnection:
     """Connect to a LanceDB database.
 
@@ -55,5 +59,5 @@ def connect(
     if isinstance(uri, str) and uri.startswith("db://"):
         if api_key is None:
             raise ValueError(f"api_key is required to connected LanceDB cloud: {uri}")
-        return RemoteDBConnection(uri, api_key, region)
+        return RemoteDBConnection(uri, api_key, region, host_override)
     return LanceDBConnection(uri)

python/lancedb/common.py

@@ -11,17 +11,18 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 from pathlib import Path
-from typing import List, Union
+from typing import Iterable, List, Union
 
 import numpy as np
-import pandas as pd
 import pyarrow as pa
 
+from .util import safe_import_pandas
+
+pd = safe_import_pandas()
+
+DATA = Union[List[dict], dict, "pd.DataFrame", pa.Table, Iterable[pa.RecordBatch]]
 VEC = Union[list, np.ndarray, pa.Array, pa.ChunkedArray]
 URI = Union[str, Path]
-
-# TODO support generator
-DATA = Union[List[dict], dict, pd.DataFrame]
 VECTOR_COLUMN_NAME = "vector"
 
 

python/lancedb/context.py

@@ -12,12 +12,13 @@
 #  limitations under the License.
 from __future__ import annotations
 
-import pandas as pd
-
 from .exceptions import MissingColumnError, MissingValueError
+from .util import safe_import_pandas
+
+pd = safe_import_pandas()
 
 
-def contextualize(raw_df: pd.DataFrame) -> Contextualizer:
+def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
     """Create a Contextualizer object for the given DataFrame.
 
     Used to create context windows. Context windows are rolling subsets of text
@@ -175,8 +176,12 @@ class Contextualizer:
         self._min_window_size = min_window_size
         return self
 
-    def to_df(self) -> pd.DataFrame:
+    def to_df(self) -> "pd.DataFrame":
         """Create the context windows and return a DataFrame."""
+        if pd is None:
+            raise ImportError(
+                "pandas is required to create context windows using lancedb"
+            )
 
         if self._text_col not in self._raw_df.columns.tolist():
             raise MissingColumnError(self._text_col)

python/lancedb/db.py

@@ -13,17 +13,18 @@
 
 from __future__ import annotations
 
-import functools
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
+from typing import Optional
 
 import pyarrow as pa
 from pyarrow import fs
 
 from .common import DATA, URI
+from .pydantic import LanceModel
 from .table import LanceTable, Table
-from .util import get_uri_location, get_uri_scheme
+from .util import fs_from_uri, get_uri_location, get_uri_scheme
 
 
 class DBConnection(ABC):
@@ -38,8 +39,8 @@ class DBConnection(ABC):
     def create_table(
         self,
         name: str,
-        data: DATA = None,
+        data: Optional[DATA] = None,
-        schema: pa.Schema = None,
+        schema: Optional[pa.Schema, LanceModel] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
@@ -51,8 +52,8 @@ class DBConnection(ABC):
         name: str
             The name of the table.
         data: list, tuple, dict, pd.DataFrame; optional
-            The data to insert into the table.
+            The data to initialize the table. User must provide at least one of `data` or `schema`.
-        schema: pyarrow.Schema; optional
+        schema: pyarrow.Schema or LanceModel; optional
             The schema of the table.
         mode: str; default "create"
             The mode to use when creating the table. Can be either "create" or "overwrite".
@@ -64,16 +65,16 @@ class DBConnection(ABC):
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
-        Note
-        ----
-        The vector index won't be created by default.
-        To create the index, call the `create_index` method on the table.
-
         Returns
         -------
         LanceTable
             A reference to the newly created table.
 
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
         Examples
         --------
 
@@ -119,7 +120,7 @@ class DBConnection(ABC):
 
         Data is converted to Arrow before being written to disk. For maximum
         control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a PyArrow table directly.
+        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
@@ -138,6 +139,30 @@ class DBConnection(ABC):
         vector: [[[1.1,1.2],[0.2,1.8]]]
         lat: [[45.5,40.1]]
         long: [[-122.7,-74.1]]
+
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]]),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32())),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema)
+        LanceTable(table4)
+
         """
         raise NotImplementedError
 
@@ -168,6 +193,13 @@ class DBConnection(ABC):
         """
         raise NotImplementedError
 
+    def drop_database(self):
+        """
+        Drop database
+        This is the same thing as dropping all the tables
+        """
+        raise NotImplementedError
+
 
 class LanceDBConnection(DBConnection):
     """
@@ -225,7 +257,7 @@ class LanceDBConnection(DBConnection):
             A list of table names.
         """
         try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
+            filesystem, path = fs_from_uri(self.uri)
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
@@ -252,122 +284,30 @@ class LanceDBConnection(DBConnection):
     def create_table(
         self,
         name: str,
-        data: DATA = None,
+        data: Optional[DATA] = None,
-        schema: pa.Schema = None,
+        schema: Optional[pa.Schema, LanceModel] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> LanceTable:
         """Create a table in the database.
 
-        Parameters
+        See
-        ----------
+        ---
-        name: str
+        DBConnection.create_table
-            The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to insert into the table.
-        schema: pyarrow.Schema; optional
-            The schema of the table.
-        mode: str; default "create"
-            The mode to use when creating the table. Can be either "create" or "overwrite".
-            By default, if the table already exists, an exception is raised.
-            If you want to overwrite the table, use mode="overwrite".
-        on_bad_vectors: str, default "error"
-            What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
-        fill_value: float
-            The value to use when filling vectors. Only used if on_bad_vectors="fill".
-
-        Note
-        ----
-        The vector index won't be created by default.
-        To create the index, call the `create_index` method on the table.
-
-        Returns
-        -------
-        LanceTable
-            A reference to the newly created table.
-
-        Examples
-        --------
-
-        Can create with list of tuples or dictionaries:
-
-        >>> import lancedb
-        >>> db = lancedb.connect("./.lancedb")
-        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
-        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
-        >>> db.create_table("my_table", data)
-        LanceTable(my_table)
-        >>> db["my_table"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        You can also pass a pandas DataFrame:
-
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({
-        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
-        ...    "lat": [45.5, 40.1],
-        ...    "long": [-122.7, -74.1]
-        ... })
-        >>> db.create_table("table2", data)
-        LanceTable(table2)
-        >>> db["table2"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        Data is converted to Arrow before being written to disk. For maximum
-        control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a PyArrow table directly.
-
-        >>> custom_schema = pa.schema([
-        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
-        ...   pa.field("lat", pa.float32()),
-        ...   pa.field("long", pa.float32())
-        ... ])
-        >>> db.create_table("table3", data, schema = custom_schema)
-        LanceTable(table3)
-        >>> db["table3"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: float
-        long: float
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
         """
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
 
-        if data is not None:
+        tbl = LanceTable.create(
-            tbl = LanceTable.create(
+            self,
-                self,
+            name,
-                name,
+            data,
-                data,
+            schema,
-                schema,
+            mode=mode,
-                mode=mode,
+            on_bad_vectors=on_bad_vectors,
-                on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
-                fill_value=fill_value,
+        )
-            )
-        else:
-            tbl = LanceTable.open(self, name)
         return tbl
 
     def open_table(self, name: str) -> LanceTable:
@@ -384,14 +324,24 @@ class LanceDBConnection(DBConnection):
         """
         return LanceTable.open(self, name)
 
-    def drop_table(self, name: str):
+    def drop_table(self, name: str, ignore_missing: bool = False):
         """Drop a table from the database.
 
         Parameters
         ----------
         name: str
             The name of the table.
+        ignore_missing: bool, default False
+            If True, ignore if the table does not exist.
         """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        try:
-        table_path = os.path.join(path, name + ".lance")
+            filesystem, path = fs_from_uri(self.uri)
-        filesystem.delete_dir(table_path)
+            table_path = os.path.join(path, name + ".lance")
+            filesystem.delete_dir(table_path)
+        except FileNotFoundError:
+            if not ignore_missing:
+                raise
+
+    def drop_database(self):
+        filesystem, path = fs_from_uri(self.uri)
+        filesystem.delete_dir(path)

python/lancedb/embeddings.py

@@ -16,15 +16,19 @@ import sys
 from typing import Callable, Union
 
 import numpy as np
-import pandas as pd
 import pyarrow as pa
 from lance.vector import vec_to_table
 from retry import retry
 
+from .util import safe_import_pandas
+
+pd = safe_import_pandas()
+DATA = Union[pa.Table, "pd.DataFrame"]
+
 
 def with_embeddings(
     func: Callable,
-    data: Union[pa.Table, pd.DataFrame],
+    data: DATA,
     column: str = "text",
     wrap_api: bool = True,
     show_progress: bool = False,
@@ -60,7 +64,7 @@ def with_embeddings(
     func = func.batch_size(batch_size)
     if show_progress:
         func = func.show_progress()
-    if isinstance(data, pd.DataFrame):
+    if pd is not None and isinstance(data, pd.DataFrame):
         data = pa.Table.from_pandas(data, preserve_index=False)
     embeddings = func(data[column].to_numpy())
     table = vec_to_table(np.array(embeddings))

python/lancedb/pydantic.py

@@ -11,27 +11,37 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Pydantic adapter for LanceDB"""
+"""Pydantic (v1 / v2) adapter for LanceDB"""
 
 from __future__ import annotations
 
 import inspect
 import sys
 import types
-from abc import ABC, abstractstaticmethod
+from abc import ABC, abstractmethod
-from typing import Any, List, Type, Union, _GenericAlias
+from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias
 
+import numpy as np
 import pyarrow as pa
 import pydantic
-from pydantic_core import CoreSchema, core_schema
+import semver
+
+PYDANTIC_VERSION = semver.Version.parse(pydantic.__version__)
+try:
+    from pydantic_core import CoreSchema, core_schema
+except ImportError:
+    if PYDANTIC_VERSION >= (2,):
+        raise
 
 
 class FixedSizeListMixin(ABC):
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def dim() -> int:
         raise NotImplementedError
 
-    @abstractstaticmethod
+    @staticmethod
+    @abstractmethod
     def value_arrow_type() -> pa.DataType:
         raise NotImplementedError
 
@@ -41,9 +51,15 @@ def vector(
 ) -> Type[FixedSizeListMixin]:
     """Pydantic Vector Type.
 
-    Note
+    !!! warning
-    ----
+        Experimental feature.
-    Experimental feature.
+
+    Parameters
+    ----------
+    dim : int
+        The dimension of the vector.
+    value_type : pyarrow.DataType, optional
+        The value type of the vector, by default pa.float32()
 
     Examples
     --------
@@ -52,13 +68,22 @@ def vector(
     >>> from lancedb.pydantic import vector
     ...
     >>> class MyModel(pydantic.BaseModel):
-    ...     vector: vector(756)
     ...     id: int
-    ...     description: str
+    ...     url: str
+    ...     embeddings: vector(768)
+    >>> schema = pydantic_to_schema(MyModel)
+    >>> assert schema == pa.schema([
+    ...     pa.field("id", pa.int64(), False),
+    ...     pa.field("url", pa.utf8(), False),
+    ...     pa.field("embeddings", pa.list_(pa.float32(), 768), False)
+    ... ])
     """
 
     # TODO: make a public parameterized type.
     class FixedSizeList(list, FixedSizeListMixin):
+        def __repr__(self):
+            return f"FixedSizeList(dim={dim})"
+
         @staticmethod
         def dim() -> int:
             return dim
@@ -80,6 +105,25 @@ def vector(
                 ),
             )
 
+        @classmethod
+        def __get_validators__(cls) -> Generator[Callable, None, None]:
+            yield cls.validate
+
+        # For pydantic v1
+        @classmethod
+        def validate(cls, v):
+            if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim:
+                raise TypeError("A list of numbers or numpy.ndarray is needed")
+            return v
+
+        if PYDANTIC_VERSION < (2, 0):
+
+            @classmethod
+            def __modify_schema__(cls, field_schema: Dict[str, Any]):
+                field_schema["items"] = {"type": "number"}
+                field_schema["maxItems"] = dim
+                field_schema["minItems"] = dim
+
     return FixedSizeList
 
 
@@ -106,11 +150,20 @@ def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
     )
 
 
-def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+if PYDANTIC_VERSION.major < 2:
-    fields = []
+
-    for name, field in model.model_fields.items():
+    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
-        fields.append(_pydantic_to_field(name, field))
+        return [
-    return fields
+            _pydantic_to_field(name, field) for name, field in model.__fields__.items()
+        ]
+
+else:
+
+    def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+        return [
+            _pydantic_to_field(name, field)
+            for name, field in model.model_fields.items()
+        ]
 
 
 def _pydantic_to_arrow_type(field: pydantic.fields.FieldInfo) -> pa.DataType:
@@ -163,7 +216,75 @@ def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema:
 
     Returns
     -------
-    A PyArrow Schema.
+    pyarrow.Schema
+
+    Examples
+    --------
+
+    >>> from typing import List, Optional
+    >>> import pydantic
+    >>> from lancedb.pydantic import pydantic_to_schema
+    ...
+    >>> class InnerModel(pydantic.BaseModel):
+    ...     a: str
+    ...     b: Optional[float]
+    >>>
+    >>> class FooModel(pydantic.BaseModel):
+    ...     id: int
+    ...     s: Optional[str] = None
+    ...     vec: List[float]
+    ...     li: List[int]
+    ...     inner: InnerModel
+    >>> schema = pydantic_to_schema(FooModel)
+    >>> assert schema == pa.schema([
+    ...     pa.field("id", pa.int64(), False),
+    ...     pa.field("s", pa.utf8(), True),
+    ...     pa.field("vec", pa.list_(pa.float64()), False),
+    ...     pa.field("li", pa.list_(pa.int64()), False),
+    ...     pa.field("inner", pa.struct([
+    ...         pa.field("a", pa.utf8(), False),
+    ...         pa.field("b", pa.float64(), True),
+    ...     ]), False),
+    ... ])
     """
     fields = _pydantic_model_to_fields(model)
     return pa.schema(fields)
+
+
+class LanceModel(pydantic.BaseModel):
+    """
+    A Pydantic Model base class that can be converted to a LanceDB Table.
+
+    Examples
+    --------
+    >>> import lancedb
+    >>> from lancedb.pydantic import LanceModel, vector
+    >>>
+    >>> class TestModel(LanceModel):
+    ...     name: str
+    ...     vector: vector(2)
+    ...
+    >>> db = lancedb.connect("/tmp")
+    >>> table = db.create_table("test", schema=TestModel.to_arrow_schema())
+    >>> table.add([
+    ...     TestModel(name="test", vector=[1.0, 2.0])
+    ... ])
+    >>> table.search([0., 0.]).limit(1).to_pydantic(TestModel)
+    [TestModel(name='test', vector=FixedSizeList(dim=2))]
+    """
+
+    @classmethod
+    def to_arrow_schema(cls):
+        """
+        Get the Arrow Schema for this model.
+        """
+        return pydantic_to_schema(cls)
+
+    @classmethod
+    def field_names(cls) -> List[str]:
+        """
+        Get the field names of this model.
+        """
+        if PYDANTIC_VERSION.major < 2:
+            return list(cls.__fields__.keys())
+        return list(cls.model_fields.keys())

python/lancedb/query.py

@@ -13,17 +13,20 @@
 
 from __future__ import annotations
 
-from typing import List, Literal, Optional, Union
+from typing import List, Literal, Optional, Type, Union
 
 import numpy as np
-import pandas as pd
 import pyarrow as pa
-from pydantic import BaseModel
+import pydantic
 
 from .common import VECTOR_COLUMN_NAME
+from .pydantic import LanceModel
+from .util import safe_import_pandas
+
+pd = safe_import_pandas()
 
 
-class Query(BaseModel):
+class Query(pydantic.BaseModel):
     """A Query"""
 
     vector_column: str = VECTOR_COLUMN_NAME
@@ -70,8 +73,8 @@ class LanceQueryBuilder:
     ...       .select(["b"])
     ...       .limit(2)
     ...       .to_df())
-       b      vector  score
+       b      vector  _distance
-    0  6  [0.4, 0.4]    0.0
+    0  6  [0.4, 0.4]        0.0
     """
 
     def __init__(
@@ -198,11 +201,11 @@ class LanceQueryBuilder:
         self._refine_factor = refine_factor
         return self
 
-    def to_df(self) -> pd.DataFrame:
+    def to_df(self) -> "pd.DataFrame":
         """
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
+        and also the "_distance" column which is the distance between the query
         vector and the returned vector.
         """
 
@@ -214,7 +217,7 @@ class LanceQueryBuilder:
         [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
 
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
+        and also the "_distance" column which is the distance between the query
         vector and the returned vectors.
         """
         vector = self._query if isinstance(self._query, list) else self._query.tolist()
@@ -226,12 +229,30 @@ class LanceQueryBuilder:
             columns=self._columns,
             nprobes=self._nprobes,
             refine_factor=self._refine_factor,
+            vector_column=self._vector_column,
         )
         return self._table._execute_query(query)
 
+    def to_pydantic(self, model: Type[LanceModel]) -> List[LanceModel]:
+        """Return the table as a list of pydantic models.
+
+        Parameters
+        ----------
+        model: Type[LanceModel]
+            The pydantic model to use.
+
+        Returns
+        -------
+        List[LanceModel]
+        """
+        return [
+            model(**{k: v for k, v in row.items() if k in model.field_names()})
+            for row in self.to_arrow().to_pylist()
+        ]
+
 
 class LanceFtsQueryBuilder(LanceQueryBuilder):
-    def to_arrow(self) -> pd.Table:
+    def to_arrow(self) -> pa.Table:
         try:
             import tantivy
         except ImportError:

python/lancedb/remote/arrow.py

@@ -0,0 +1,22 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import pyarrow as pa
+
+
+def to_ipc_binary(table: pa.Table) -> bytes:
+    """Serialize a PyArrow Table to IPC binary."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()

python/lancedb/remote/client.py

@@ -13,7 +13,7 @@
 
 
 import functools
-from typing import Any, Callable, Dict, Union
+from typing import Any, Callable, Dict, Optional, Union
 
 import aiohttp
 import attr
@@ -24,6 +24,8 @@ from lancedb.common import Credential
 from lancedb.remote import VectorQuery, VectorQueryResult
 from lancedb.remote.errors import LanceDBClientError
 
+ARROW_STREAM_CONTENT_TYPE = "application/vnd.apache.arrow.stream"
+
 
 def _check_not_closed(f):
     @functools.wraps(f)
@@ -46,11 +48,16 @@ class RestfulLanceDBClient:
     db_name: str
     region: str
     api_key: Credential
+    host_override: Optional[str] = attr.field(default=None)
+
     closed: bool = attr.field(default=False, init=False)
 
     @functools.cached_property
     def session(self) -> aiohttp.ClientSession:
-        url = f"https://{self.db_name}.{self.region}.api.lancedb.com"
+        url = (
+            self.host_override
+            or f"https://{self.db_name}.{self.region}.api.lancedb.com"
+        )
         return aiohttp.ClientSession(url)
 
     async def close(self):
@@ -59,9 +66,14 @@ class RestfulLanceDBClient:
 
     @functools.cached_property
     def headers(self) -> Dict[str, str]:
-        return {
+        headers = {
             "x-api-key": self.api_key,
         }
+        if self.region == "local":  # Local test mode
+            headers["Host"] = f"{self.db_name}.{self.region}.api.lancedb.com"
+        if self.host_override:
+            headers["x-lancedb-database"] = self.db_name
+        return headers
 
     @staticmethod
     async def _check_status(resp: aiohttp.ClientResponse):
@@ -85,7 +97,12 @@ class RestfulLanceDBClient:
         """Send a GET request and returns the deserialized response payload."""
         if isinstance(params, BaseModel):
             params: Dict[str, Any] = params.dict(exclude_none=True)
-        async with self.session.get(uri, params=params, headers=self.headers) as resp:
+        async with self.session.get(
+            uri,
+            params=params,
+            headers=self.headers,
+            timeout=aiohttp.ClientTimeout(total=30),
+        ) as resp:
             await self._check_status(resp)
             return await resp.json()
 
@@ -93,8 +110,11 @@ class RestfulLanceDBClient:
     async def post(
         self,
         uri: str,
-        data: Union[Dict[str, Any], BaseModel],
+        data: Optional[Union[Dict[str, Any], BaseModel, bytes]] = None,
+        params: Optional[Dict[str, Any]] = None,
+        content_type: Optional[str] = None,
         deserialize: Callable = lambda resp: resp.json(),
+        request_id: Optional[str] = None,
     ) -> Dict[str, Any]:
         """Send a POST request and returns the deserialized response payload.
 
@@ -103,14 +123,28 @@ class RestfulLanceDBClient:
         uri : str
             The uri to send the POST request to.
         data: Union[Dict[str, Any], BaseModel]
+        request_id: Optional[str]
+            Optional client side request id to be sent in the request headers.
 
         """
         if isinstance(data, BaseModel):
             data: Dict[str, Any] = data.dict(exclude_none=True)
+        if isinstance(data, bytes):
+            req_kwargs = {"data": data}
+        else:
+            req_kwargs = {"json": data}
+
+        headers = self.headers.copy()
+        if content_type is not None:
+            headers["content-type"] = content_type
+        if request_id is not None:
+            headers["x-request-id"] = request_id
         async with self.session.post(
             uri,
-            json=data,
+            headers=headers,
-            headers=self.headers,
+            params=params,
+            timeout=aiohttp.ClientTimeout(total=30),
+            **req_kwargs,
         ) as resp:
             resp: aiohttp.ClientResponse = resp
             await self._check_status(resp)
@@ -119,11 +153,13 @@ class RestfulLanceDBClient:
     @_check_not_closed
     async def list_tables(self):
         """List all tables in the database."""
-        json = await self.get("/1/table/", {})
+        json = await self.get("/v1/table/", {})
         return json["tables"]
 
     @_check_not_closed
     async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
         """Query a table."""
-        tbl = await self.post(f"/1/table/{table_name}/", query, deserialize=_read_ipc)
+        tbl = await self.post(
+            f"/v1/table/{table_name}/query/", query, deserialize=_read_ipc
+        )
         return VectorQueryResult(tbl)

python/lancedb/remote/db.py

@@ -12,29 +12,39 @@
 #  limitations under the License.
 
 import asyncio
-from typing import List
+import uuid
+from typing import List, Optional
 from urllib.parse import urlparse
 
 import pyarrow as pa
 
 from lancedb.common import DATA
 from lancedb.db import DBConnection
-from lancedb.table import Table
+from lancedb.table import Table, _sanitize_data
 
-from .client import RestfulLanceDBClient
+from .arrow import to_ipc_binary
+from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
 
 
 class RemoteDBConnection(DBConnection):
     """A connection to a remote LanceDB database."""
 
-    def __init__(self, db_url: str, api_key: str, region: str):
+    def __init__(
+        self,
+        db_url: str,
+        api_key: str,
+        region: str,
+        host_override: Optional[str] = None,
+    ):
         """Connect to a remote LanceDB database."""
         parsed = urlparse(db_url)
         if parsed.scheme != "db":
             raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
         self.db_name = parsed.netloc
         self.api_key = api_key
-        self._client = RestfulLanceDBClient(self.db_name, region, api_key)
+        self._client = RestfulLanceDBClient(
+            self.db_name, region, api_key, host_override
+        )
         try:
             self._loop = asyncio.get_running_loop()
         except RuntimeError:
@@ -71,8 +81,45 @@ class RemoteDBConnection(DBConnection):
         name: str,
         data: DATA = None,
         schema: pa.Schema = None,
-        mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> Table:
-        raise NotImplementedError
+        if data is None and schema is None:
+            raise ValueError("Either data or schema must be provided.")
+        if data is not None:
+            data = _sanitize_data(
+                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            )
+        else:
+            if schema is None:
+                raise ValueError("Either data or schema must be provided")
+            data = pa.Table.from_pylist([], schema=schema)
+
+        from .table import RemoteTable
+
+        data = to_ipc_binary(data)
+        request_id = uuid.uuid4().hex
+
+        self._loop.run_until_complete(
+            self._client.post(
+                f"/v1/table/{name}/create/",
+                data=data,
+                request_id=request_id,
+                content_type=ARROW_STREAM_CONTENT_TYPE,
+            )
+        )
+        return RemoteTable(self, name)
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        self._loop.run_until_complete(
+            self._client.post(
+                f"/v1/table/{name}/drop/",
+            )
+        )

python/lancedb/remote/table.py

@@ -11,14 +11,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import uuid
+from functools import cached_property
 from typing import Union
 
 import pyarrow as pa
+from lance import json_to_schema
 
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
-from ..query import LanceQueryBuilder, Query
+from ..query import LanceQueryBuilder
-from ..table import Query, Table
+from ..table import Query, Table, _sanitize_data
+from .arrow import to_ipc_binary
+from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
 
 
@@ -28,13 +33,27 @@ class RemoteTable(Table):
         self._name = name
 
     def __repr__(self) -> str:
-        return f"RemoteTable({self._conn.db_name}.{self.name})"
+        return f"RemoteTable({self._conn.db_name}.{self._name})"
 
+    @cached_property
     def schema(self) -> pa.Schema:
-        raise NotImplementedError
+        """Return the schema of the table."""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+        )
+        schema = json_to_schema(resp["schema"])
+        return schema
 
     def to_arrow(self) -> pa.Table:
-        raise NotImplementedError
+        """Return the table as an Arrow table."""
+        raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
+
+    def to_pandas(self):
+        """Return the table as a Pandas DataFrame.
+
+        Intercept `to_arrow()` for better error message.
+        """
+        return NotImplementedError("to_pandas() is not supported on the LanceDB cloud")
 
     def create_index(
         self,
@@ -53,7 +72,21 @@ class RemoteTable(Table):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> int:
-        raise NotImplementedError
+        data = _sanitize_data(
+            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
+        payload = to_ipc_binary(data)
+
+        request_id = uuid.uuid4().hex
+
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(
+                f"/v1/table/{self._name}/insert/",
+                data=payload,
+                params={"request_id": request_id, "mode": mode},
+                content_type=ARROW_STREAM_CONTENT_TYPE,
+            )
+        )
 
     def search(
         self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
@@ -63,3 +96,6 @@ class RemoteTable(Table):
     def _execute_query(self, query: Query) -> pa.Table:
         result = self._conn._client.query(self._name, query)
         return self._conn._loop.run_until_complete(result).to_arrow()
+
+    def delete(self, predicate: str):
+        raise NotImplementedError

python/lancedb/schema.py

@@ -12,10 +12,6 @@
 #  limitations under the License.
 
 """Schema related utilities."""
-
-import json
-from typing import Any, Dict, Type
-
 import pyarrow as pa
 
 
@@ -43,247 +39,3 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
     ... ])
     """
     return pa.list_(value_type, dimension)
-
-
-def _type_to_dict(dt: pa.DataType) -> Dict[str, Any]:
-    if pa.types.is_boolean(dt):
-        return {"type": "boolean"}
-    elif pa.types.is_int8(dt):
-        return {"type": "int8"}
-    elif pa.types.is_int16(dt):
-        return {"type": "int16"}
-    elif pa.types.is_int32(dt):
-        return {"type": "int32"}
-    elif pa.types.is_int64(dt):
-        return {"type": "int64"}
-    elif pa.types.is_uint8(dt):
-        return {"type": "uint8"}
-    elif pa.types.is_uint16(dt):
-        return {"type": "uint16"}
-    elif pa.types.is_uint32(dt):
-        return {"type": "uint32"}
-    elif pa.types.is_uint64(dt):
-        return {"type": "uint64"}
-    elif pa.types.is_float16(dt):
-        return {"type": "float16"}
-    elif pa.types.is_float32(dt):
-        return {"type": "float32"}
-    elif pa.types.is_float64(dt):
-        return {"type": "float64"}
-    elif pa.types.is_date32(dt):
-        return {"type": f"date32"}
-    elif pa.types.is_date64(dt):
-        return {"type": f"date64"}
-    elif pa.types.is_time32(dt):
-        return {"type": f"time32:{dt.unit}"}
-    elif pa.types.is_time64(dt):
-        return {"type": f"time64:{dt.unit}"}
-    elif pa.types.is_timestamp(dt):
-        return {"type": f"timestamp:{dt.unit}:{dt.tz if dt.tz is not None else ''}"}
-    elif pa.types.is_string(dt):
-        return {"type": "string"}
-    elif pa.types.is_binary(dt):
-        return {"type": "binary"}
-    elif pa.types.is_large_string(dt):
-        return {"type": "large_string"}
-    elif pa.types.is_large_binary(dt):
-        return {"type": "large_binary"}
-    elif pa.types.is_fixed_size_binary(dt):
-        return {"type": "fixed_size_binary", "width": dt.byte_width}
-    elif pa.types.is_fixed_size_list(dt):
-        return {
-            "type": "fixed_size_list",
-            "width": dt.list_size,
-            "value_type": _type_to_dict(dt.value_type),
-        }
-    elif pa.types.is_list(dt):
-        return {
-            "type": "list",
-            "value_type": _type_to_dict(dt.value_type),
-        }
-    elif pa.types.is_struct(dt):
-        return {
-            "type": "struct",
-            "fields": [_field_to_dict(dt.field(i)) for i in range(dt.num_fields)],
-        }
-    elif pa.types.is_dictionary(dt):
-        return {
-            "type": "dictionary",
-            "index_type": _type_to_dict(dt.index_type),
-            "value_type": _type_to_dict(dt.value_type),
-        }
-    # TODO: support extension types
-
-    raise TypeError(f"Unsupported type: {dt}")
-
-
-def _field_to_dict(field: pa.field) -> Dict[str, Any]:
-    ret = {
-        "name": field.name,
-        "type": _type_to_dict(field.type),
-        "nullable": field.nullable,
-    }
-    if field.metadata is not None:
-        ret["metadata"] = field.metadata
-    return ret
-
-
-def schema_to_dict(schema: pa.Schema) -> Dict[str, Any]:
-    """Convert a PyArrow [Schema](pyarrow.Schema) to a dictionary.
-
-    Parameters
-    ----------
-    schema : pa.Schema
-        The PyArrow Schema to convert
-
-    Returns
-    -------
-    A dict of the data type.
-
-    Examples
-    --------
-
-    >>> import pyarrow as pa
-    >>> import lancedb
-    >>> schema = pa.schema(
-    ...     [
-    ...         pa.field("id", pa.int64()),
-    ...         pa.field("vector", lancedb.vector(512), nullable=False),
-    ...         pa.field(
-    ...             "struct",
-    ...             pa.struct(
-    ...             [
-    ...                 pa.field("a", pa.utf8()),
-    ...                 pa.field("b", pa.float32()),
-    ...             ]
-    ...         ),
-    ...         True,
-    ...     ),
-    ...     ],
-    ...     metadata={"key": "value"},
-    ... )
-    >>> json_schema = schema_to_dict(schema)
-    >>> assert json_schema == {
-    ...     "fields": [
-    ...     {"name": "id", "type": {"type": "int64"}, "nullable": True},
-    ...     {
-    ...         "name": "vector",
-    ...         "type": {
-    ...             "type": "fixed_size_list",
-    ...             "value_type": {"type": "float32"},
-    ...             "width": 512,
-    ...         },
-    ...        "nullable": False,
-    ...    },
-    ...    {
-    ...         "name": "struct",
-    ...         "type": {
-    ...             "type": "struct",
-    ...             "fields": [
-    ...                 {"name": "a", "type": {"type": "string"}, "nullable": True},
-    ...                 {"name": "b", "type": {"type": "float32"}, "nullable": True},
-    ...            ],
-    ...         },
-    ...         "nullable": True,
-    ...     },
-    ...     ],
-    ...     "metadata": {"key": "value"},
-    ... }
-
-    """
-    fields = []
-    for name in schema.names:
-        field = schema.field(name)
-        fields.append(_field_to_dict(field))
-    json_schema = {
-        "fields": fields,
-        "metadata": {
-            k.decode("utf-8"): v.decode("utf-8") for (k, v) in schema.metadata.items()
-        }
-        if schema.metadata is not None
-        else {},
-    }
-    return json_schema
-
-
-def _dict_to_type(dt: Dict[str, Any]) -> pa.DataType:
-    type_name = dt["type"]
-    try:
-        return {
-            "boolean": pa.bool_(),
-            "int8": pa.int8(),
-            "int16": pa.int16(),
-            "int32": pa.int32(),
-            "int64": pa.int64(),
-            "uint8": pa.uint8(),
-            "uint16": pa.uint16(),
-            "uint32": pa.uint32(),
-            "uint64": pa.uint64(),
-            "float16": pa.float16(),
-            "float32": pa.float32(),
-            "float64": pa.float64(),
-            "string": pa.string(),
-            "binary": pa.binary(),
-            "large_string": pa.large_string(),
-            "large_binary": pa.large_binary(),
-            "date32": pa.date32(),
-            "date64": pa.date64(),
-        }[type_name]
-    except KeyError:
-        pass
-
-    if type_name == "fixed_size_binary":
-        return pa.binary(dt["width"])
-    elif type_name == "fixed_size_list":
-        return pa.list_(_dict_to_type(dt["value_type"]), dt["width"])
-    elif type_name == "list":
-        return pa.list_(_dict_to_type(dt["value_type"]))
-    elif type_name == "struct":
-        fields = []
-        for field in dt["fields"]:
-            fields.append(_dict_to_field(field))
-        return pa.struct(fields)
-    elif type_name == "dictionary":
-        return pa.dictionary(
-            _dict_to_type(dt["index_type"]), _dict_to_type(dt["value_type"])
-        )
-    elif type_name.startswith("time32:"):
-        return pa.time32(type_name.split(":")[1])
-    elif type_name.startswith("time64:"):
-        return pa.time64(type_name.split(":")[1])
-    elif type_name.startswith("timestamp:"):
-        fields = type_name.split(":")
-        unit = fields[1]
-        tz = fields[2] if len(fields) > 2 else None
-        return pa.timestamp(unit, tz)
-    raise TypeError(f"Unsupported type: {dt}")
-
-
-def _dict_to_field(field: Dict[str, Any]) -> pa.Field:
-    name = field["name"]
-    nullable = field["nullable"] if "nullable" in field else True
-    dt = _dict_to_type(field["type"])
-    metadata = field.get("metadata", None)
-    return pa.field(name, dt, nullable, metadata)
-
-
-def dict_to_schema(json: Dict[str, Any]) -> pa.Schema:
-    """Reconstruct a PyArrow Schema from a JSON dict.
-
-    Parameters
-    ----------
-    json : Dict[str, Any]
-        The JSON dict to reconstruct Schema from.
-
-    Returns
-    -------
-    A PyArrow Schema.
-    """
-    fields = []
-    for field in json["fields"]:
-        fields.append(_dict_to_field(field))
-    metadata = {
-        k.encode("utf-8"): v.encode("utf-8")
-        for (k, v) in json.get("metadata", {}).items()
-    }
-    return pa.schema(fields, metadata)

python/lancedb/table.py

@@ -13,38 +13,50 @@
 
 from __future__ import annotations
 
+import inspect
 import os
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import List, Union
+from typing import Iterable, List, Union
 
 import lance
 import numpy as np
-import pandas as pd
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow.fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
+from .pydantic import LanceModel
 from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
+from .util import fs_from_uri, safe_import_pandas
+
+pd = safe_import_pandas()
 
 
 def _sanitize_data(data, schema, on_bad_vectors, fill_value):
     if isinstance(data, list):
+        # convert to list of dict if data is a bunch of LanceModels
+        if isinstance(data[0], LanceModel):
+            schema = data[0].__class__.to_arrow_schema()
+            data = [dict(d) for d in data]
         data = pa.Table.from_pylist(data)
         data = _sanitize_schema(
             data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
     if isinstance(data, dict):
         data = vec_to_table(data)
-    if isinstance(data, pd.DataFrame):
+    if pd is not None and isinstance(data, pd.DataFrame):
-        data = pa.Table.from_pandas(data)
+        data = pa.Table.from_pandas(data, preserve_index=False)
         data = _sanitize_schema(
             data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
-    if not isinstance(data, pa.Table):
+        # Do not serialize Pandas metadata
+        metadata = data.schema.metadata if data.schema.metadata is not None else {}
+        metadata = {k: v for k, v in metadata.items() if k != b"pandas"}
+        schema = data.schema.with_metadata(metadata)
+        data = pa.Table.from_arrays(data.columns, schema=schema)
+    if not isinstance(data, (pa.Table, Iterable)):
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
 
@@ -74,28 +86,28 @@ class Table(ABC):
     Can append new data with [Table.add()][lancedb.table.Table.add].
 
     >>> table.add([{"vector": [0.5, 1.3], "b": 4}])
-    2
 
     Can query the table with [Table.search][lancedb.table.Table.search].
 
     >>> table.search([0.4, 0.4]).select(["b"]).to_df()
-       b      vector  score
+       b      vector  _distance
-    0  4  [0.5, 1.3]   0.82
+    0  4  [0.5, 1.3]       0.82
-    1  2  [1.1, 1.2]   1.13
+    1  2  [1.1, 1.2]       1.13
 
     Search queries are much faster when an index is created. See
     [Table.create_index][lancedb.table.Table.create_index].
     """
 
+    @property
     @abstractmethod
     def schema(self) -> pa.Schema:
-        """Return the [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
         this [Table](Table)
 
         """
         raise NotImplementedError
 
-    def to_pandas(self) -> pd.DataFrame:
+    def to_pandas(self):
         """Return the table as a pandas DataFrame.
 
         Returns
@@ -151,7 +163,7 @@ class Table(ABC):
         mode: str = "append",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-    ) -> int:
+    ):
         """Add more data to the [Table](Table).
 
         Parameters
@@ -167,10 +179,6 @@ class Table(ABC):
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
-        Returns
-        -------
-        int
-            The number of vectors in the table.
         """
         raise NotImplementedError
 
@@ -193,7 +201,7 @@ class Table(ABC):
         LanceQueryBuilder
             A query builder object representing the query.
             Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         raise NotImplementedError
@@ -202,6 +210,51 @@ class Table(ABC):
     def _execute_query(self, query: Query) -> pa.Table:
         pass
 
+    @abstractmethod
+    def delete(self, where: str):
+        """Delete rows from the table.
+
+        This can be used to delete a single row, many rows, all rows, or
+        sometimes no rows (if your predicate matches nothing).
+
+        Parameters
+        ----------
+        where: str
+            The SQL where clause to use when deleting rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+        >>> table.delete("x = 2")
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  3  [5.0, 6.0]
+
+        If you have a list of values to delete, you can combine them into a
+        stringified list and use the `IN` operator:
+
+        >>> to_remove = [1, 5]
+        >>> to_remove = ", ".join([str(v) for v in to_remove])
+        >>> to_remove
+        '1, 5'
+        >>> table.delete(f"x IN ({to_remove})")
+        >>> table.to_pandas()
+           x      vector
+        0  3  [5.0, 6.0]
+        """
+        raise NotImplementedError
+
 
 class LanceTable(Table):
     """
@@ -262,7 +315,6 @@ class LanceTable(Table):
                vector    type
         0  [1.1, 0.9]  vector
         >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
-        2
         >>> table.version
         2
         >>> table.checkout(1)
@@ -289,7 +341,7 @@ class LanceTable(Table):
         """Return the first n rows of the table."""
         return self._dataset.head(n)
 
-    def to_pandas(self) -> pd.DataFrame:
+    def to_pandas(self) -> "pd.DataFrame":
         """Return the table as a pandas DataFrame.
 
         Returns
@@ -364,7 +416,7 @@ class LanceTable(Table):
         mode: str = "append",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-    ) -> int:
+    ):
         """Add data to the table.
 
         Parameters
@@ -389,9 +441,8 @@ class LanceTable(Table):
         data = _sanitize_data(
             data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
-        lance.write_dataset(data, self._dataset_uri, mode=mode)
+        lance.write_dataset(data, self._dataset_uri, schema=self.schema, mode=mode)
         self._reset_dataset()
-        return len(self)
 
     def search(
         self, query: Union[VEC, str], vector_column_name=VECTOR_COLUMN_NAME
@@ -411,7 +462,7 @@ class LanceTable(Table):
         LanceQueryBuilder
             A query builder object representing the query.
             Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         if isinstance(query, str):
@@ -462,7 +513,7 @@ class LanceTable(Table):
         data: list-of-dict, dict, pd.DataFrame, default None
             The data to insert into the table.
             At least one of `data` or `schema` must be provided.
-        schema: dict, optional
+        schema: pa.Schema or LanceModel, optional
             The schema of the table. If not provided, the schema is inferred from the data.
             At least one of `data` or `schema` must be provided.
         mode: str, default "create"
@@ -475,6 +526,8 @@ class LanceTable(Table):
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
         """
         tbl = LanceTable(db, name)
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            schema = schema.to_arrow_schema()
         if data is not None:
             data = _sanitize_data(
                 data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
@@ -483,44 +536,21 @@ class LanceTable(Table):
             if schema is None:
                 raise ValueError("Either data or schema must be provided")
             data = pa.Table.from_pylist([], schema=schema)
-        lance.write_dataset(data, tbl._dataset_uri, mode=mode)
+        lance.write_dataset(data, tbl._dataset_uri, schema=schema, mode=mode)
         return LanceTable(db, name)
 
     @classmethod
     def open(cls, db, name):
         tbl = cls(db, name)
-        if not os.path.exists(tbl._dataset_uri):
+        fs, path = fs_from_uri(tbl._dataset_uri)
+        file_info = fs.get_file_info(path)
+        if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(
                 f"Table {name} does not exist. Please first call db.create_table({name}, data)"
             )
         return tbl
 
     def delete(self, where: str):
-        """Delete rows from the table.
-
-        Parameters
-        ----------
-        where: str
-            The SQL where clause to use when deleting rows.
-
-        Examples
-        --------
-        >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-        >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", data)
-        >>> table.to_pandas()
-           x      vector
-        0  1  [1.0, 2.0]
-        1  2  [3.0, 4.0]
-        2  3  [5.0, 6.0]
-        >>> table.delete("x = 2")
-        >>> table.to_pandas()
-           x      vector
-        0  1  [1.0, 2.0]
-        1  3  [5.0, 6.0]
-        """
         self._dataset.delete(where)
 
     def _execute_query(self, query: Query) -> pa.Table:

python/lancedb/util.py

@@ -11,8 +11,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import os
+from typing import Tuple
 from urllib.parse import urlparse
 
+import pyarrow.fs as pa_fs
+
 
 def get_uri_scheme(uri: str) -> str:
     """
@@ -59,3 +63,24 @@ def get_uri_location(uri: str) -> str:
         return parsed.path
     else:
         return parsed.netloc + parsed.path
+
+
+def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
+    """
+    Get a PyArrow FileSystem from a URI, handling extra environment variables.
+    """
+    if get_uri_scheme(uri) == "s3":
+        fs = pa_fs.S3FileSystem(endpoint_override=os.environ.get("AWS_ENDPOINT"))
+        path = get_uri_location(uri)
+        return fs, path
+
+    return pa_fs.FileSystem.from_uri(uri)
+
+
+def safe_import_pandas():
+    try:
+        import pandas as pd
+
+        return pd
+    except ImportError:
+        return None

python/pyproject.toml

@@ -1,11 +1,18 @@
 [project]
 name = "lancedb"
-version = "0.1.10"
+version = "0.2.0"
-dependencies = ["pylance~=0.5.0", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic>=2", "attr"]
+dependencies = [
-description = "lancedb"
+    "pylance==0.6.1",
-authors = [
+    "ratelimiter",
-    { name = "LanceDB Devs", email = "dev@lancedb.com" },
+    "retry",
+    "tqdm",
+    "aiohttp",
+    "pydantic",
+    "attr",
+    "semver>=3.0"
 ]
+description = "lancedb"
+authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
 license = { file = "LICENSE" }
 readme = "README.md"
 requires-python = ">=3.8"
@@ -36,19 +43,13 @@ classifiers = [
 repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
-tests = [
+tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio"]
-    "pytest", "pytest-mock", "pytest-asyncio"
+dev = ["ruff", "pre-commit", "black"]
-]
+docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
-dev = [
-    "ruff", "pre-commit", "black"
-]
-docs = [
-    "mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"
-]
 
 [build-system]
-requires = [
+requires = ["setuptools", "wheel"]
-  "setuptools",
-  "wheel",
-]
 build-backend = "setuptools.build_meta"
+
+[tool.isort]
+profile = "black"

python/tests/test_db.py

@@ -13,9 +13,11 @@
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
 import pytest
 
 import lancedb
+from lancedb.pydantic import LanceModel
 
 
 def test_basic(tmp_path):
@@ -75,6 +77,37 @@ def test_ingest_pd(tmp_path):
     assert db.open_table("test").name == db["test"].name
 
 
+def test_ingest_record_batch_iterator(tmp_path):
+    def batch_reader():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table(
+        "test",
+        batch_reader(),
+        schema=pa.schema(
+            [
+                pa.field("vector", pa.list_(pa.float32())),
+                pa.field("item", pa.utf8()),
+                pa.field("price", pa.float32()),
+            ]
+        ),
+    )
+
+    tbl_len = len(tbl)
+    tbl.add(batch_reader())
+    assert len(tbl) == tbl_len * 2
+    assert len(tbl.list_versions()) == 2
+
+
 def test_create_mode(tmp_path):
     db = lancedb.connect(tmp_path)
     data = pd.DataFrame(
@@ -122,6 +155,51 @@ def test_delete_table(tmp_path):
     db.create_table("test", data=data)
     assert db.table_names() == ["test"]
 
+    # dropping a table that does not exist should pass
+    # if ignore_missing=True
+    db.drop_table("does_not_exist", ignore_missing=True)
+
+
+def test_drop_database(tmp_path):
+    db = lancedb.connect(tmp_path)
+    data = pd.DataFrame(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", "bar"],
+            "price": [10.0, 20.0],
+        }
+    )
+    new_data = pd.DataFrame(
+        {
+            "vector": [[5.1, 4.1], [5.9, 10.5]],
+            "item": ["kiwi", "avocado"],
+            "price": [12.0, 17.0],
+        }
+    )
+    db.create_table("test", data=data)
+    with pytest.raises(Exception):
+        db.create_table("test", data=data)
+
+    assert db.table_names() == ["test"]
+
+    db.create_table("new_test", data=new_data)
+    db.drop_database()
+    assert db.table_names() == []
+
+    # it should pass when no tables are present
+    db.create_table("test", data=new_data)
+    db.drop_table("test")
+    assert db.table_names() == []
+    db.drop_database()
+    assert db.table_names() == []
+
+    # creating an empty database with schema
+    schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), list_size=2))])
+    db.create_table("empty_table", schema=schema)
+    # dropping a empty database should pass
+    db.drop_database()
+    assert db.table_names() == []
+
 
 def test_empty_or_nonexistent_table(tmp_path):
     db = lancedb.connect(tmp_path)
@@ -131,6 +209,15 @@ def test_empty_or_nonexistent_table(tmp_path):
     with pytest.raises(Exception):
         db.open_table("does_not_exist")
 
+    schema = pa.schema([pa.field("a", pa.int64(), nullable=False)])
+    test = db.create_table("test", schema=schema)
+
+    class TestModel(LanceModel):
+        a: int
+
+    test2 = db.create_table("test2", schema=TestModel)
+    assert test.schema == test2.schema
+
 
 def test_replace_index(tmp_path):
     db = lancedb.connect(uri=tmp_path)

python/tests/test_fts.py

@@ -66,7 +66,7 @@ def test_search_index(tmp_path, table):
     results = ldb.fts.search_index(index, query="puppy", limit=10)
     assert len(results) == 2
     assert len(results[0]) == 10  # row_ids
-    assert len(results[1]) == 10  # scores
+    assert len(results[1]) == 10  # _distance
 
 
 def test_create_index_from_table(tmp_path, table):

python/tests/test_pydantic.py

@@ -20,7 +20,7 @@ import pyarrow as pa
 import pydantic
 import pytest
 
-from lancedb.pydantic import pydantic_to_schema, vector
+from lancedb.pydantic import PYDANTIC_VERSION, LanceModel, pydantic_to_schema, vector
 
 
 @pytest.mark.skipif(
@@ -111,10 +111,16 @@ def test_fixed_size_list_field():
         li: List[int]
 
     data = TestModel(vec=list(range(16)), li=[1, 2, 3])
-    assert json.loads(data.model_dump_json()) == {
+    if PYDANTIC_VERSION >= (2,):
-        "vec": list(range(16)),
+        assert json.loads(data.model_dump_json()) == {
-        "li": [1, 2, 3],
+            "vec": list(range(16)),
-    }
+            "li": [1, 2, 3],
+        }
+    else:
+        assert data.dict() == {
+            "vec": list(range(16)),
+            "li": [1, 2, 3],
+        }
 
     schema = pydantic_to_schema(TestModel)
     assert schema == pa.schema(
@@ -124,7 +130,11 @@ def test_fixed_size_list_field():
         ]
     )
 
-    json_schema = TestModel.model_json_schema()
+    if PYDANTIC_VERSION >= (2,):
+        json_schema = TestModel.model_json_schema()
+    else:
+        json_schema = TestModel.schema()
+
     assert json_schema == {
         "properties": {
             "vec": {
@@ -153,3 +163,13 @@ def test_fixed_size_list_validation():
         TestModel(vec=range(7))
 
     TestModel(vec=range(8))
+
+
+def test_lance_model():
+    class TestModel(LanceModel):
+        vec: vector(16)
+        li: List[int]
+
+    schema = pydantic_to_schema(TestModel)
+    assert schema == TestModel.to_arrow_schema()
+    assert TestModel.field_names() == ["vec", "li"]

python/tests/test_query.py

@@ -20,6 +20,7 @@ import pyarrow as pa
 import pytest
 
 from lancedb.db import LanceDBConnection
+from lancedb.pydantic import LanceModel, vector
 from lancedb.query import LanceQueryBuilder, Query
 from lancedb.table import LanceTable
 
@@ -64,6 +65,24 @@ def table(tmp_path) -> MockTable:
     return MockTable(tmp_path)
 
 
+def test_cast(table):
+    class TestModel(LanceModel):
+        vector: vector(2)
+        id: int
+        str_field: str
+        float_field: float
+
+    q = LanceQueryBuilder(table, [0, 0], "vector").limit(1)
+    results = q.to_pydantic(TestModel)
+    assert len(results) == 1
+    r0 = results[0]
+    assert isinstance(r0, TestModel)
+    assert r0.id == 1
+    assert r0.vector == [1, 2]
+    assert r0.str_field == "a"
+    assert r0.float_field == 1.0
+
+
 def test_query_builder(table):
     df = LanceQueryBuilder(table, [0, 0], "vector").limit(1).select(["id"]).to_df()
     assert df["id"].values[0] == 1
@@ -89,11 +108,11 @@ def test_query_builder_with_metric(table):
         .limit(1)
         .to_df()
     )
-    assert df_cosine.score[0] == pytest.approx(
+    assert df_cosine._distance[0] == pytest.approx(
         cosine_distance(query, df_cosine.vector[0]),
         abs=1e-6,
     )
-    assert 0 <= df_cosine.score[0] <= 1
+    assert 0 <= df_cosine._distance[0] <= 1
 
 
 def test_query_builder_with_different_vector_column():
@@ -119,6 +138,7 @@ def test_query_builder_with_different_vector_column():
             columns=["b"],
             nprobes=20,
             refine_factor=None,
+            vector_column="foo_vector",
         )
     )
 

python/tests/test_schema.py

@@ -1,109 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import pyarrow as pa
-
-import lancedb
-from lancedb.schema import dict_to_schema, schema_to_dict
-
-
-def test_schema_to_dict():
-    schema = pa.schema(
-        [
-            pa.field("id", pa.int64()),
-            pa.field("vector", lancedb.vector(512), nullable=False),
-            pa.field(
-                "struct",
-                pa.struct(
-                    [
-                        pa.field("a", pa.utf8()),
-                        pa.field("b", pa.float32()),
-                    ]
-                ),
-                True,
-            ),
-            pa.field("d", pa.dictionary(pa.int64(), pa.utf8()), False),
-        ],
-        metadata={"key": "value"},
-    )
-
-    json_schema = schema_to_dict(schema)
-    assert json_schema == {
-        "fields": [
-            {"name": "id", "type": {"type": "int64"}, "nullable": True},
-            {
-                "name": "vector",
-                "type": {
-                    "type": "fixed_size_list",
-                    "value_type": {"type": "float32"},
-                    "width": 512,
-                },
-                "nullable": False,
-            },
-            {
-                "name": "struct",
-                "type": {
-                    "type": "struct",
-                    "fields": [
-                        {"name": "a", "type": {"type": "string"}, "nullable": True},
-                        {"name": "b", "type": {"type": "float32"}, "nullable": True},
-                    ],
-                },
-                "nullable": True,
-            },
-            {
-                "name": "d",
-                "type": {
-                    "type": "dictionary",
-                    "index_type": {"type": "int64"},
-                    "value_type": {"type": "string"},
-                },
-                "nullable": False,
-            },
-        ],
-        "metadata": {"key": "value"},
-    }
-
-    actual_schema = dict_to_schema(json_schema)
-    assert actual_schema == schema
-
-
-def test_temporal_types():
-    schema = pa.schema(
-        [
-            pa.field("t32", pa.time32("s")),
-            pa.field("t32ms", pa.time32("ms")),
-            pa.field("t64", pa.time64("ns")),
-            pa.field("ts", pa.timestamp("s")),
-            pa.field("ts_us_tz", pa.timestamp("us", tz="America/New_York")),
-        ],
-    )
-    json_schema = schema_to_dict(schema)
-
-    assert json_schema == {
-        "fields": [
-            {"name": "t32", "type": {"type": "time32:s"}, "nullable": True},
-            {"name": "t32ms", "type": {"type": "time32:ms"}, "nullable": True},
-            {"name": "t64", "type": {"type": "time64:ns"}, "nullable": True},
-            {"name": "ts", "type": {"type": "timestamp:s:"}, "nullable": True},
-            {
-                "name": "ts_us_tz",
-                "type": {"type": "timestamp:us:America/New_York"},
-                "nullable": True,
-            },
-        ],
-        "metadata": {},
-    }
-
-    actual_schema = dict_to_schema(json_schema)
-    assert actual_schema == schema

python/tests/test_table.py

@@ -13,15 +13,16 @@
 
 import functools
 from pathlib import Path
+from typing import List
 from unittest.mock import PropertyMock, patch
 
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
-from lance.vector import vec_to_table
 
 from lancedb.db import LanceDBConnection
+from lancedb.pydantic import LanceModel, vector
 from lancedb.table import LanceTable
 
 
@@ -135,12 +136,23 @@ def test_add(db):
     _add(table, schema)
 
 
+def test_add_pydantic_model(db):
+    class TestModel(LanceModel):
+        vector: vector(16)
+        li: List[int]
+
+    data = TestModel(vector=list(range(16)), li=[1, 2, 3])
+    table = LanceTable.create(db, "test", data=[data])
+    assert len(table) == 1
+    assert table.schema == TestModel.to_arrow_schema()
+
+
 def _add(table, schema):
     # table = LanceTable(db, "test")
     assert len(table) == 2
 
-    count = table.add([{"vector": [6.3, 100.5], "item": "new", "price": 30.0}])
+    table.add([{"vector": [6.3, 100.5], "item": "new", "price": 30.0}])
-    assert count == 3
+    assert len(table) == 3
 
     expected = pa.Table.from_arrays(
         [

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.13"
+version = "0.1.19"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -13,12 +13,15 @@ crate-type = ["cdylib"]
 arrow-array = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
+conv = "0.3.3"
 once_cell = "1"
 futures = "0.3"
+half = { workspace = true }
 lance = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }
 object_store = { workspace = true, features = ["aws"] }
+snafu = { workspace = true }
 async-trait = "0"
 env_logger = "0"

rust/ffi/node/src/arrow.rs

@@ -13,27 +13,30 @@
 // limitations under the License.
 
 use std::io::Cursor;
+use std::ops::Deref;
 use std::sync::Arc;
 
 use arrow_array::cast::as_list_array;
-use arrow_array::{Array, FixedSizeListArray, RecordBatch};
+use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch};
 use arrow_ipc::reader::FileReader;
+use arrow_ipc::writer::FileWriter;
 use arrow_schema::{DataType, Field, Schema};
 use lance::arrow::{FixedSizeListArrayExt, RecordBatchExt};
+use vectordb::table::VECTOR_COLUMN_NAME;
 
-pub(crate) fn convert_record_batch(record_batch: RecordBatch) -> RecordBatch {
+use crate::error::{MissingColumnSnafu, Result};
-    let column = record_batch
+use snafu::prelude::*;
-        .column_by_name("vector")
+
-        .cloned()
+pub(crate) fn convert_record_batch(record_batch: RecordBatch) -> Result<RecordBatch> {
-        .expect("vector column is missing");
+    let column = get_column(VECTOR_COLUMN_NAME, &record_batch)?;
-    // TODO: we should just consume the underlaying js buffer in the future instead of this arrow around a bunch of times
+
+    // TODO: we should just consume the underlying js buffer in the future instead of this arrow around a bunch of times
     let arr = as_list_array(column.as_ref());
     let list_size = arr.values().len() / record_batch.num_rows();
-    let r =
+    let r = FixedSizeListArray::try_new_from_values(arr.values().to_owned(), list_size as i32)?;
-        FixedSizeListArray::try_new_from_values(arr.values().to_owned(), list_size as i32).unwrap();
 
     let schema = Arc::new(Schema::new(vec![Field::new(
-        "vector",
+        VECTOR_COLUMN_NAME,
         DataType::FixedSizeList(
             Arc::new(Field::new("item", DataType::Float32, true)),
             list_size as i32,
@@ -41,22 +44,42 @@ pub(crate) fn convert_record_batch(record_batch: RecordBatch) -> RecordBatch {
         true,
     )]));
 
-    let mut new_batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(r)]).unwrap();
+    let mut new_batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(r)])?;
 
     if record_batch.num_columns() > 1 {
-        let rb = record_batch.drop_column("vector").unwrap();
+        let rb = record_batch.drop_column(VECTOR_COLUMN_NAME)?;
-        new_batch = new_batch.merge(&rb).unwrap();
+        new_batch = new_batch.merge(&rb)?;
     }
-    new_batch
+    Ok(new_batch)
 }
 
-pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Vec<RecordBatch> {
+fn get_column(column_name: &str, record_batch: &RecordBatch) -> Result<ArrayRef> {
+    record_batch
+        .column_by_name(column_name)
+        .cloned()
+        .context(MissingColumnSnafu { name: column_name })
+}
+
+pub(crate) fn arrow_buffer_to_record_batch(slice: &[u8]) -> Result<Vec<RecordBatch>> {
     let mut batches: Vec<RecordBatch> = Vec::new();
-    let fr = FileReader::try_new(Cursor::new(slice), None);
+    let file_reader = FileReader::try_new(Cursor::new(slice), None)?;
-    let file_reader = fr.unwrap();
     for b in file_reader {
-        let record_batch = convert_record_batch(b.unwrap());
+        let record_batch = convert_record_batch(b?)?;
         batches.push(record_batch);
     }
-    batches
+    Ok(batches)
+}
+
+pub(crate) fn record_batch_to_buffer(batches: Vec<RecordBatch>) -> Result<Vec<u8>> {
+    if batches.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let schema = batches.get(0).unwrap().schema();
+    let mut fr = FileWriter::try_new(Vec::new(), schema.deref())?;
+    for batch in batches.iter() {
+        fr.write(batch)?
+    }
+    fr.finish()?;
+    Ok(fr.into_inner()?)
 }

rust/ffi/node/src/error.rs

@@ -0,0 +1,96 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow_schema::ArrowError;
+use neon::context::Context;
+use neon::prelude::NeonResult;
+use snafu::Snafu;
+
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub(crate)))]
+pub enum Error {
+    #[snafu(display("column '{name}' is missing"))]
+    MissingColumn { name: String },
+    #[snafu(display("{name}: {message}"))]
+    RangeError { name: String, message: String },
+    #[snafu(display("{index_type} is not a valid index type"))]
+    InvalidIndexType { index_type: String },
+
+    #[snafu(display("{message}"))]
+    LanceDB { message: String },
+    #[snafu(display("{message}"))]
+    Neon { message: String },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<vectordb::error::Error> for Error {
+    fn from(e: vectordb::error::Error) -> Self {
+        Self::LanceDB {
+            message: e.to_string(),
+        }
+    }
+}
+
+impl From<lance::Error> for Error {
+    fn from(e: lance::Error) -> Self {
+        Self::LanceDB {
+            message: e.to_string(),
+        }
+    }
+}
+
+impl From<ArrowError> for Error {
+    fn from(value: ArrowError) -> Self {
+        Self::LanceDB {
+            message: value.to_string(),
+        }
+    }
+}
+
+impl From<neon::result::Throw> for Error {
+    fn from(value: neon::result::Throw) -> Self {
+        Self::Neon {
+            message: value.to_string(),
+        }
+    }
+}
+
+impl<T> From<std::sync::mpsc::SendError<T>> for Error {
+    fn from(value: std::sync::mpsc::SendError<T>) -> Self {
+        Self::Neon {
+            message: value.to_string(),
+        }
+    }
+}
+
+/// ResultExt is used to transform a [`Result`] into a [`NeonResult`],
+/// so it can be returned as a JavaScript error
+/// Copied from [Neon](https://github.com/neon-bindings/neon/blob/4c2e455a9e6814f1ba0178616d63caec7f4df317/crates/neon/src/result/mod.rs#L88)
+pub trait ResultExt<T> {
+    fn or_throw<'a, C: Context<'a>>(self, cx: &mut C) -> NeonResult<T>;
+}
+
+/// Implement ResultExt for the std Result so it can be used any Result type
+impl<T, E> ResultExt<T> for std::result::Result<T, E>
+where
+    E: std::fmt::Display,
+{
+    fn or_throw<'a, C: Context<'a>>(self, cx: &mut C) -> NeonResult<T> {
+        match self {
+            Ok(value) => Ok(value),
+            Err(error) => cx.throw_error(error.to_string()),
+        }
+    }
+}

rust/ffi/node/src/index/vector.rs

@@ -12,40 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::convert::TryFrom;
-
 use lance::index::vector::ivf::IvfBuildParams;
 use lance::index::vector::pq::PQBuildParams;
 use lance::index::vector::MetricType;
 use neon::context::FunctionContext;
 use neon::prelude::*;
+use std::convert::TryFrom;
 
 use vectordb::index::vector::{IvfPQIndexBuilder, VectorIndexBuilder};
 
-use crate::{runtime, JsTable};
+use crate::error::Error::InvalidIndexType;
+use crate::error::ResultExt;
+use crate::neon_ext::js_object_ext::JsObjectExt;
+use crate::runtime;
+use crate::table::JsTable;
 
 pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
     let index_params = cx.argument::<JsObject>(0)?;
-    let index_params_builder = get_index_params_builder(&mut cx, index_params).unwrap();
+    let index_params_builder = get_index_params_builder(&mut cx, index_params).or_throw(&mut cx)?;
 
     let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
 
     let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
+    let channel = cx.channel();
+    let mut table = js_table.table.clone();
 
-    rt.block_on(async move {
+    rt.spawn(async move {
-        let add_result = table
+        let idx_result = table.create_index(&index_params_builder).await;
-            .lock()
-            .unwrap()
-            .create_index(&index_params_builder)
-            .await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            add_result
+            idx_result.or_throw(&mut cx)?;
-                .map(|_| cx.undefined())
+            Ok(cx.boxed(JsTable::from(table)))
-                .or_else(|err| cx.throw_error(err.to_string()))
         });
     });
     Ok(promise)
@@ -54,27 +52,21 @@ pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsP
 fn get_index_params_builder(
     cx: &mut FunctionContext,
     obj: Handle<JsObject>,
-) -> Result<impl VectorIndexBuilder, String> {
+) -> crate::error::Result<impl VectorIndexBuilder> {
-    let idx_type = obj
+    let idx_type = obj.get::<JsString, _, _>(cx, "type")?.value(cx);
-        .get::<JsString, _, _>(cx, "type")
-        .map_err(|t| t.to_string())?
-        .value(cx);
 
     match idx_type.as_str() {
         "ivf_pq" => {
             let mut index_builder: IvfPQIndexBuilder = IvfPQIndexBuilder::new();
             let mut pq_params = PQBuildParams::default();
 
-            obj.get_opt::<JsString, _, _>(cx, "column")
+            obj.get_opt::<JsString, _, _>(cx, "column")?
-                .map_err(|t| t.to_string())?
                 .map(|s| index_builder.column(s.value(cx)));
 
-            obj.get_opt::<JsString, _, _>(cx, "index_name")
+            obj.get_opt::<JsString, _, _>(cx, "index_name")?
-                .map_err(|t| t.to_string())?
                 .map(|s| index_builder.index_name(s.value(cx)));
 
-            obj.get_opt::<JsString, _, _>(cx, "metric_type")
+            obj.get_opt::<JsString, _, _>(cx, "metric_type")?
-                .map_err(|t| t.to_string())?
                 .map(|s| MetricType::try_from(s.value(cx).as_str()))
                 .map(|mt| {
                     let metric_type = mt.unwrap();
@@ -82,15 +74,8 @@ fn get_index_params_builder(
                     pq_params.metric_type = metric_type;
                 });
 
-            let num_partitions = obj
+            let num_partitions = obj.get_opt_usize(cx, "num_partitions")?;
-                .get_opt::<JsNumber, _, _>(cx, "num_partitions")
+            let max_iters = obj.get_opt_usize(cx, "max_iters")?;
-                .map_err(|t| t.to_string())?
-                .map(|s| s.value(cx) as usize);
-
-            let max_iters = obj
-                .get_opt::<JsNumber, _, _>(cx, "max_iters")
-                .map_err(|t| t.to_string())?
-                .map(|s| s.value(cx) as usize);
 
             num_partitions.map(|np| {
                 let max_iters = max_iters.unwrap_or(50);
@@ -102,32 +87,28 @@ fn get_index_params_builder(
                 index_builder.ivf_params(ivf_params)
             });
 
-            obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")
+            obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")?
-                .map_err(|t| t.to_string())?
                 .map(|s| pq_params.use_opq = s.value(cx));
 
-            obj.get_opt::<JsNumber, _, _>(cx, "num_sub_vectors")
+            obj.get_opt_usize(cx, "num_sub_vectors")?
-                .map_err(|t| t.to_string())?
+                .map(|s| pq_params.num_sub_vectors = s);
-                .map(|s| pq_params.num_sub_vectors = s.value(cx) as usize);
 
-            obj.get_opt::<JsNumber, _, _>(cx, "num_bits")
+            obj.get_opt_usize(cx, "num_bits")?
-                .map_err(|t| t.to_string())?
+                .map(|s| pq_params.num_bits = s);
-                .map(|s| pq_params.num_bits = s.value(cx) as usize);
 
-            obj.get_opt::<JsNumber, _, _>(cx, "max_iters")
+            obj.get_opt_usize(cx, "max_iters")?
-                .map_err(|t| t.to_string())?
+                .map(|s| pq_params.max_iters = s);
-                .map(|s| pq_params.max_iters = s.value(cx) as usize);
 
-            obj.get_opt::<JsNumber, _, _>(cx, "max_opq_iters")
+            obj.get_opt_usize(cx, "max_opq_iters")?
-                .map_err(|t| t.to_string())?
+                .map(|s| pq_params.max_opq_iters = s);
-                .map(|s| pq_params.max_opq_iters = s.value(cx) as usize);
 
-            obj.get_opt::<JsBoolean, _, _>(cx, "replace")
+            obj.get_opt::<JsBoolean, _, _>(cx, "replace")?
-                .map_err(|t| t.to_string())?
                 .map(|s| index_builder.replace(s.value(cx)));
 
             Ok(index_builder)
         }
-        t => Err(format!("{} is not a valid index type", t).to_string()),
+        index_type => Err(InvalidIndexType {
+            index_type: index_type.into(),
+        }),
     }
 }

rust/ffi/node/src/lib.rs

@@ -12,34 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::sync::Arc;
-use std::convert::TryFrom;
-use std::ops::Deref;
-use std::sync::{Arc, Mutex};
 
-use arrow_array::{Float32Array, RecordBatchIterator};
-use arrow_ipc::writer::FileWriter;
 use async_trait::async_trait;
-use futures::{TryFutureExt, TryStreamExt};
-use lance::dataset::{ReadParams, WriteMode, WriteParams};
-use lance::index::vector::MetricType;
 use lance::io::object_store::ObjectStoreParams;
 use neon::prelude::*;
-use neon::types::buffer::TypedArray;
 use object_store::aws::{AwsCredential, AwsCredentialProvider};
 use object_store::CredentialProvider;
 use once_cell::sync::OnceCell;
 use tokio::runtime::Runtime;
 
 use vectordb::database::Database;
-use vectordb::error::Error;
+use vectordb::table::ReadParams;
-use vectordb::table::{OpenTableParams, Table};
 
-use crate::arrow::arrow_buffer_to_record_batch;
+use crate::error::ResultExt;
+use crate::query::JsQuery;
+use crate::table::JsTable;
 
 mod arrow;
 mod convert;
+mod error;
 mod index;
+mod neon_ext;
+mod query;
+mod table;
 
 struct JsDatabase {
     database: Arc<Database>,
@@ -47,14 +43,8 @@ struct JsDatabase {
 
 impl Finalize for JsDatabase {}
 
-struct JsTable {
-    table: Arc<Mutex<Table>>,
-}
-
-impl Finalize for JsTable {}
-
 // TODO: object_store didn't export this type so I copied it.
-// Make a requiest to object_store to export this type
+// Make a request to object_store to export this type
 #[derive(Debug)]
 pub struct StaticCredentialProvider<T> {
     credential: Arc<T>,
@@ -86,7 +76,7 @@ fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
 
     LOG.get_or_init(|| env_logger::init());
 
-    RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
+    RUNTIME.get_or_try_init(|| Runtime::new().or_throw(cx))
 }
 
 fn database_new(mut cx: FunctionContext) -> JsResult<JsPromise> {
@@ -101,7 +91,7 @@ fn database_new(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
         deferred.settle_with(&channel, move |mut cx| {
             let db = JsDatabase {
-                database: Arc::new(database.or_else(|err| cx.throw_error(err.to_string()))?),
+                database: Arc::new(database.or_throw(&mut cx)?),
             };
             Ok(cx.boxed(db))
         });
@@ -123,7 +113,7 @@ fn database_table_names(mut cx: FunctionContext) -> JsResult<JsPromise> {
         let tables_rst = database.table_names().await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            let tables = tables_rst.or_else(|err| cx.throw_error(err.to_string()))?;
+            let tables = tables_rst.or_throw(&mut cx)?;
             let table_names = convert::vec_str_to_array(&tables, &mut cx);
             table_names
         });
@@ -177,7 +167,7 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
         Err(err) => return err,
     };
 
-    let param = ReadParams {
+    let params = ReadParams {
         store_options: Some(ObjectStoreParams {
             aws_credentials: aws_creds,
             ..ObjectStoreParams::default()
@@ -191,20 +181,11 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let (deferred, promise) = cx.promise();
     rt.spawn(async move {
-        let table_rst = database
+        let table_rst = database.open_table_with_params(&table_name, &params).await;
-            .open_table_with_params(
-                &table_name,
-                OpenTableParams {
-                    open_table_params: param,
-                },
-            )
-            .await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            let table = Arc::new(Mutex::new(
+            let js_table = JsTable::from(table_rst.or_throw(&mut cx)?);
-                table_rst.or_else(|err| cx.throw_error(err.to_string()))?,
+            Ok(cx.boxed(js_table))
-            ));
-            Ok(cx.boxed(JsTable { table }))
         });
     });
     Ok(promise)
@@ -224,245 +205,24 @@ fn database_drop_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
     rt.spawn(async move {
         let result = database.drop_table(&table_name).await;
         deferred.settle_with(&channel, move |mut cx| {
-            result.or_else(|err| cx.throw_error(err.to_string()))?;
+            result.or_throw(&mut cx)?;
             Ok(cx.null())
         });
     });
     Ok(promise)
 }
 
-fn table_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let query_obj = cx.argument::<JsObject>(0)?;
-
-    let limit = query_obj
-        .get::<JsNumber, _, _>(&mut cx, "_limit")?
-        .value(&mut cx);
-    let select = query_obj
-        .get_opt::<JsArray, _, _>(&mut cx, "_select")?
-        .map(|arr| {
-            let js_array = arr.deref();
-            let mut projection_vec: Vec<String> = Vec::new();
-            for i in 0..js_array.len(&mut cx) {
-                let entry: Handle<JsString> = js_array.get(&mut cx, i).unwrap();
-                projection_vec.push(entry.value(&mut cx));
-            }
-            projection_vec
-        });
-    let filter = query_obj
-        .get_opt::<JsString, _, _>(&mut cx, "_filter")?
-        .map(|s| s.value(&mut cx));
-    let refine_factor = query_obj
-        .get_opt::<JsNumber, _, _>(&mut cx, "_refineFactor")?
-        .map(|s| s.value(&mut cx))
-        .map(|i| i as u32);
-    let nprobes = query_obj
-        .get::<JsNumber, _, _>(&mut cx, "_nprobes")?
-        .value(&mut cx) as usize;
-    let metric_type = query_obj
-        .get_opt::<JsString, _, _>(&mut cx, "_metricType")?
-        .map(|s| s.value(&mut cx))
-        .map(|s| MetricType::try_from(s.as_str()).unwrap());
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
-    let query_vector = query_obj.get::<JsArray, _, _>(&mut cx, "_queryVector")?;
-    let query = convert::js_array_to_vec(query_vector.deref(), &mut cx);
-
-    rt.spawn(async move {
-        let builder = table
-            .lock()
-            .unwrap()
-            .search(Float32Array::from(query))
-            .limit(limit as usize)
-            .refine_factor(refine_factor)
-            .nprobes(nprobes)
-            .filter(filter)
-            .metric_type(metric_type)
-            .select(select);
-        let record_batch_stream = builder.execute();
-        let results = record_batch_stream
-            .and_then(|stream| stream.try_collect::<Vec<_>>().map_err(Error::from))
-            .await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let results = results.or_else(|err| cx.throw_error(err.to_string()))?;
-            let vector: Vec<u8> = Vec::new();
-
-            if results.is_empty() {
-                return cx.buffer(0);
-            }
-
-            let schema = results.get(0).unwrap().schema();
-            let mut fr = FileWriter::try_new(vector, schema.deref())
-                .or_else(|err| cx.throw_error(err.to_string()))?;
-
-            for batch in results.iter() {
-                fr.write(batch)
-                    .or_else(|err| cx.throw_error(err.to_string()))?;
-            }
-            fr.finish().or_else(|err| cx.throw_error(err.to_string()))?;
-            let buf = fr
-                .into_inner()
-                .or_else(|err| cx.throw_error(err.to_string()))?;
-            Ok(JsBuffer::external(&mut cx, buf))
-        });
-    });
-    Ok(promise)
-}
-
-fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let db = cx
-        .this()
-        .downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
-    let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
-    let buffer = cx.argument::<JsBuffer>(1)?;
-    let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx));
-    let schema = batches[0].schema();
-
-    // Write mode
-    let mode = match cx.argument::<JsString>(2)?.value(&mut cx).as_str() {
-        "overwrite" => WriteMode::Overwrite,
-        "append" => WriteMode::Append,
-        "create" => WriteMode::Create,
-        _ => return cx.throw_error("Table::create only supports 'overwrite' and 'create' modes"),
-    };
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let database = db.database.clone();
-
-    let aws_creds = match get_aws_creds(&mut cx, 3) {
-        Ok(creds) => creds,
-        Err(err) => return err,
-    };
-
-    let params = WriteParams {
-        store_params: Some(ObjectStoreParams {
-            aws_credentials: aws_creds,
-            ..ObjectStoreParams::default()
-        }),
-        mode: mode,
-        ..WriteParams::default()
-    };
-
-    rt.block_on(async move {
-        let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
-        let table_rst = database
-            .create_table(&table_name, batch_reader, Some(params))
-            .await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let table = Arc::new(Mutex::new(
-                table_rst.or_else(|err| cx.throw_error(err.to_string()))?,
-            ));
-            Ok(cx.boxed(JsTable { table }))
-        });
-    });
-    Ok(promise)
-}
-
-fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let write_mode_map: HashMap<&str, WriteMode> = HashMap::from([
-        ("create", WriteMode::Create),
-        ("append", WriteMode::Append),
-        ("overwrite", WriteMode::Overwrite),
-    ]);
-
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let buffer = cx.argument::<JsBuffer>(0)?;
-    let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
-    let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx));
-    let schema = batches[0].schema();
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
-    let write_mode = write_mode_map.get(write_mode.as_str()).cloned();
-
-    let aws_creds = match get_aws_creds(&mut cx, 2) {
-        Ok(creds) => creds,
-        Err(err) => return err,
-    };
-
-    let params = WriteParams {
-        store_params: Some(ObjectStoreParams {
-            aws_credentials: aws_creds,
-            ..ObjectStoreParams::default()
-        }),
-        mode: write_mode.unwrap_or(WriteMode::Append),
-        ..WriteParams::default()
-    };
-
-    rt.block_on(async move {
-        let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
-        let add_result = table.lock().unwrap().add(batch_reader, Some(params)).await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let _added = add_result.or_else(|err| cx.throw_error(err.to_string()))?;
-            Ok(cx.boolean(true))
-        });
-    });
-    Ok(promise)
-}
-
-fn table_count_rows(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
-
-    rt.block_on(async move {
-        let num_rows_result = table.lock().unwrap().count_rows().await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let num_rows = num_rows_result.or_else(|err| cx.throw_error(err.to_string()))?;
-            Ok(cx.number(num_rows as f64))
-        });
-    });
-    Ok(promise)
-}
-
-fn table_delete(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-
-    let (deferred, promise) = cx.promise();
-    let table = js_table.table.clone();
-
-    let predicate = cx.argument::<JsString>(0)?.value(&mut cx);
-
-    let delete_result = rt.block_on(async move { table.lock().unwrap().delete(&predicate).await });
-
-    deferred.settle_with(&channel, move |mut cx| {
-        delete_result.or_else(|err| cx.throw_error(err.to_string()))?;
-        Ok(cx.undefined())
-    });
-
-    Ok(promise)
-}
-
 #[neon::main]
 fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("databaseNew", database_new)?;
     cx.export_function("databaseTableNames", database_table_names)?;
     cx.export_function("databaseOpenTable", database_open_table)?;
     cx.export_function("databaseDropTable", database_drop_table)?;
-    cx.export_function("tableSearch", table_search)?;
+    cx.export_function("tableSearch", JsQuery::js_search)?;
-    cx.export_function("tableCreate", table_create)?;
+    cx.export_function("tableCreate", JsTable::js_create)?;
-    cx.export_function("tableAdd", table_add)?;
+    cx.export_function("tableAdd", JsTable::js_add)?;
-    cx.export_function("tableCountRows", table_count_rows)?;
+    cx.export_function("tableCountRows", JsTable::js_count_rows)?;
-    cx.export_function("tableDelete", table_delete)?;
+    cx.export_function("tableDelete", JsTable::js_delete)?;
     cx.export_function(
         "tableCreateVectorIndex",
         index::vector::table_create_vector_index,

rust/ffi/node/src/neon_ext.rs

@@ -0,0 +1,15 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod js_object_ext;

rust/ffi/node/src/neon_ext/js_object_ext.rs

@@ -0,0 +1,82 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::{Error, Result};
+use neon::prelude::*;
+
+// extends neon's [JsObject] with helper functions to extract properties
+pub trait JsObjectExt {
+    fn get_opt_u32(&self, cx: &mut FunctionContext, key: &str) -> Result<Option<u32>>;
+    fn get_usize(&self, cx: &mut FunctionContext, key: &str) -> Result<usize>;
+    fn get_opt_usize(&self, cx: &mut FunctionContext, key: &str) -> Result<Option<usize>>;
+}
+
+impl JsObjectExt for JsObject {
+    fn get_opt_u32(&self, cx: &mut FunctionContext, key: &str) -> Result<Option<u32>> {
+        let val_opt = self
+            .get_opt::<JsNumber, _, _>(cx, key)?
+            .map(|s| f64_to_u32_safe(s.value(cx), key));
+        val_opt.transpose()
+    }
+
+    fn get_usize(&self, cx: &mut FunctionContext, key: &str) -> Result<usize> {
+        let val = self.get::<JsNumber, _, _>(cx, key)?.value(cx);
+        f64_to_usize_safe(val, key)
+    }
+
+    fn get_opt_usize(&self, cx: &mut FunctionContext, key: &str) -> Result<Option<usize>> {
+        let val_opt = self
+            .get_opt::<JsNumber, _, _>(cx, key)?
+            .map(|s| f64_to_usize_safe(s.value(cx), key));
+        val_opt.transpose()
+    }
+}
+
+fn f64_to_u32_safe(n: f64, key: &str) -> Result<u32> {
+    use conv::*;
+
+    n.approx_as::<u32>().map_err(|e| match e {
+        FloatError::NegOverflow(_) => Error::RangeError {
+            name: key.into(),
+            message: "must be > 0".to_string(),
+        },
+        FloatError::PosOverflow(_) => Error::RangeError {
+            name: key.into(),
+            message: format!("must be < {}", u32::MAX),
+        },
+        FloatError::NotANumber(_) => Error::RangeError {
+            name: key.into(),
+            message: "not a valid number".to_string(),
+        },
+    })
+}
+
+fn f64_to_usize_safe(n: f64, key: &str) -> Result<usize> {
+    use conv::*;
+
+    n.approx_as::<usize>().map_err(|e| match e {
+        FloatError::NegOverflow(_) => Error::RangeError {
+            name: key.into(),
+            message: "must be > 0".to_string(),
+        },
+        FloatError::PosOverflow(_) => Error::RangeError {
+            name: key.into(),
+            message: format!("must be < {}", usize::MAX),
+        },
+        FloatError::NotANumber(_) => Error::RangeError {
+            name: key.into(),
+            message: "not a valid number".to_string(),
+        },
+    })
+}

rust/ffi/node/src/query.rs

@@ -0,0 +1,84 @@
+use std::convert::TryFrom;
+use std::ops::Deref;
+
+use arrow_array::Float32Array;
+use futures::{TryFutureExt, TryStreamExt};
+use lance::index::vector::MetricType;
+use neon::context::FunctionContext;
+use neon::handle::Handle;
+use neon::prelude::*;
+
+use crate::arrow::record_batch_to_buffer;
+use crate::error::ResultExt;
+use crate::neon_ext::js_object_ext::JsObjectExt;
+use crate::table::JsTable;
+use crate::{convert, runtime};
+
+pub(crate) struct JsQuery {}
+
+impl JsQuery {
+    pub(crate) fn js_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let query_obj = cx.argument::<JsObject>(0)?;
+
+        let limit = query_obj
+            .get::<JsNumber, _, _>(&mut cx, "_limit")?
+            .value(&mut cx);
+        let select = query_obj
+            .get_opt::<JsArray, _, _>(&mut cx, "_select")?
+            .map(|arr| {
+                let js_array = arr.deref();
+                let mut projection_vec: Vec<String> = Vec::new();
+                for i in 0..js_array.len(&mut cx) {
+                    let entry: Handle<JsString> = js_array.get(&mut cx, i).unwrap();
+                    projection_vec.push(entry.value(&mut cx));
+                }
+                projection_vec
+            });
+        let filter = query_obj
+            .get_opt::<JsString, _, _>(&mut cx, "_filter")?
+            .map(|s| s.value(&mut cx));
+        let refine_factor = query_obj
+            .get_opt_u32(&mut cx, "_refineFactor")
+            .or_throw(&mut cx)?;
+        let nprobes = query_obj.get_usize(&mut cx, "_nprobes").or_throw(&mut cx)?;
+        let metric_type = query_obj
+            .get_opt::<JsString, _, _>(&mut cx, "_metricType")?
+            .map(|s| s.value(&mut cx))
+            .map(|s| MetricType::try_from(s.as_str()).unwrap());
+
+        let rt = runtime(&mut cx)?;
+
+        let (deferred, promise) = cx.promise();
+        let channel = cx.channel();
+        let query_vector = query_obj.get::<JsArray, _, _>(&mut cx, "_queryVector")?;
+        let query = convert::js_array_to_vec(query_vector.deref(), &mut cx);
+        let table = js_table.table.clone();
+
+        rt.spawn(async move {
+            let builder = table
+                .search(Float32Array::from(query))
+                .limit(limit as usize)
+                .refine_factor(refine_factor)
+                .nprobes(nprobes)
+                .filter(filter)
+                .metric_type(metric_type)
+                .select(select);
+            let record_batch_stream = builder.execute();
+            let results = record_batch_stream
+                .and_then(|stream| {
+                    stream
+                        .try_collect::<Vec<_>>()
+                        .map_err(vectordb::error::Error::from)
+                })
+                .await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let results = results.or_throw(&mut cx)?;
+                let buffer = record_batch_to_buffer(results).or_throw(&mut cx)?;
+                Ok(JsBuffer::external(&mut cx, buffer))
+            });
+        });
+        Ok(promise)
+    }
+}

rust/ffi/node/src/table.rs

@@ -0,0 +1,174 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow_array::RecordBatchIterator;
+use lance::dataset::{WriteMode, WriteParams};
+use lance::io::object_store::ObjectStoreParams;
+
+use crate::arrow::arrow_buffer_to_record_batch;
+use neon::prelude::*;
+use neon::types::buffer::TypedArray;
+use vectordb::Table;
+
+use crate::error::ResultExt;
+use crate::{get_aws_creds, runtime, JsDatabase};
+
+pub(crate) struct JsTable {
+    pub table: Table,
+}
+
+impl Finalize for JsTable {}
+
+impl From<Table> for JsTable {
+    fn from(table: Table) -> Self {
+        JsTable { table }
+    }
+}
+
+impl JsTable {
+    pub(crate) fn js_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let db = cx
+            .this()
+            .downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
+        let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
+        let buffer = cx.argument::<JsBuffer>(1)?;
+        let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
+        let schema = batches[0].schema();
+
+        // Write mode
+        let mode = match cx.argument::<JsString>(2)?.value(&mut cx).as_str() {
+            "overwrite" => WriteMode::Overwrite,
+            "append" => WriteMode::Append,
+            "create" => WriteMode::Create,
+            _ => {
+                return cx.throw_error("Table::create only supports 'overwrite' and 'create' modes")
+            }
+        };
+
+        let rt = runtime(&mut cx)?;
+        let channel = cx.channel();
+
+        let (deferred, promise) = cx.promise();
+        let database = db.database.clone();
+
+        let aws_creds = match get_aws_creds(&mut cx, 3) {
+            Ok(creds) => creds,
+            Err(err) => return err,
+        };
+
+        let params = WriteParams {
+            store_params: Some(ObjectStoreParams {
+                aws_credentials: aws_creds,
+                ..ObjectStoreParams::default()
+            }),
+            mode: mode,
+            ..WriteParams::default()
+        };
+
+        rt.spawn(async move {
+            let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+            let table_rst = database
+                .create_table(&table_name, batch_reader, Some(params))
+                .await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let table = table_rst.or_throw(&mut cx)?;
+                Ok(cx.boxed(JsTable::from(table)))
+            });
+        });
+        Ok(promise)
+    }
+
+    pub(crate) fn js_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let buffer = cx.argument::<JsBuffer>(0)?;
+        let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
+
+        let batches = arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
+        let schema = batches[0].schema();
+
+        let rt = runtime(&mut cx)?;
+        let channel = cx.channel();
+        let mut table = js_table.table.clone();
+
+        let (deferred, promise) = cx.promise();
+        let write_mode = match write_mode.as_str() {
+            "create" => WriteMode::Create,
+            "append" => WriteMode::Append,
+            "overwrite" => WriteMode::Overwrite,
+            s => return cx.throw_error(format!("invalid write mode {}", s)),
+        };
+        let aws_creds = match get_aws_creds(&mut cx, 2) {
+            Ok(creds) => creds,
+            Err(err) => return err,
+        };
+
+        let params = WriteParams {
+            store_params: Some(ObjectStoreParams {
+                aws_credentials: aws_creds,
+                ..ObjectStoreParams::default()
+            }),
+            mode: write_mode,
+            ..WriteParams::default()
+        };
+
+        rt.spawn(async move {
+            let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+            let add_result = table.add(batch_reader, Some(params)).await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let _added = add_result.or_throw(&mut cx)?;
+                Ok(cx.boxed(JsTable::from(table)))
+            });
+        });
+        Ok(promise)
+    }
+
+    pub(crate) fn js_count_rows(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let rt = runtime(&mut cx)?;
+        let (deferred, promise) = cx.promise();
+        let channel = cx.channel();
+        let table = js_table.table.clone();
+
+        rt.spawn(async move {
+            let num_rows_result = table.count_rows().await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let num_rows = num_rows_result.or_throw(&mut cx)?;
+                Ok(cx.number(num_rows as f64))
+            });
+        });
+        Ok(promise)
+    }
+
+    pub(crate) fn js_delete(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let rt = runtime(&mut cx)?;
+        let (deferred, promise) = cx.promise();
+        let predicate = cx.argument::<JsString>(0)?.value(&mut cx);
+        let channel = cx.channel();
+        let mut table = js_table.table.clone();
+
+        rt.spawn(async move {
+            let delete_result = table.delete(&predicate).await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                delete_result.or_throw(&mut cx)?;
+                Ok(cx.boxed(JsTable::from(table)))
+            })
+        });
+        Ok(promise)
+    }
+}

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.1.13"
+version = "0.1.19"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -12,7 +12,8 @@ arrow-array = { workspace = true }
 arrow-data = { workspace = true }
 arrow-schema = { workspace = true }
 object_store = { workspace = true }
-snafu = "0.7.4"
+snafu = { workspace = true }
+half = { workspace = true }
 lance = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 

rust/vectordb/src/database.rs

@@ -20,13 +20,16 @@ use lance::dataset::WriteParams;
 use lance::io::object_store::ObjectStore;
 use snafu::prelude::*;
 
-use crate::error::{CreateDirSnafu, Result};
+use crate::error::{CreateDirSnafu, InvalidTableNameSnafu, Result};
-use crate::table::{OpenTableParams, Table};
+use crate::table::{ReadParams, Table};
+
+pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 pub struct Database {
     object_store: ObjectStore,
 
     pub(crate) uri: String,
+    pub(crate) base_path: object_store::path::Path,
 }
 
 const LANCE_EXTENSION: &str = "lance";
@@ -43,12 +46,13 @@ impl Database {
     ///
     /// * A [Database] object.
     pub async fn connect(uri: &str) -> Result<Database> {
-        let (object_store, _) = ObjectStore::from_uri(uri).await?;
+        let (object_store, base_path) = ObjectStore::from_uri(uri).await?;
         if object_store.is_local() {
             Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
         }
         Ok(Database {
             uri: uri.to_string(),
+            base_path,
             object_store,
         })
     }
@@ -57,7 +61,7 @@ impl Database {
     fn try_create_dir(path: &str) -> core::result::Result<(), std::io::Error> {
         let path = Path::new(path);
         if !path.try_exists()? {
-            create_dir_all(&path)?;
+            create_dir_all(path)?;
         }
         Ok(())
     }
@@ -70,23 +74,18 @@ impl Database {
     pub async fn table_names(&self) -> Result<Vec<String>> {
         let f = self
             .object_store
-            .read_dir(self.uri.as_str())
+            .read_dir(self.base_path.clone())
             .await?
             .iter()
-            .map(|fname| Path::new(fname))
+            .map(Path::new)
             .filter(|path| {
                 let is_lance = path
                     .extension()
-                    .map(|e| e.to_str().map(|e| e == LANCE_EXTENSION))
+                    .and_then(|e| e.to_str())
-                    .flatten();
+                    .map(|e| e == LANCE_EXTENSION);
                 is_lance.unwrap_or(false)
             })
-            .map(|p| {
+            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-                p.file_stem()
-                    .map(|s| s.to_str().map(|s| String::from(s)))
-                    .flatten()
-            })
-            .flatten()
             .collect();
         Ok(f)
     }
@@ -103,7 +102,8 @@ impl Database {
         batches: impl RecordBatchReader + Send + 'static,
         params: Option<WriteParams>,
     ) -> Result<Table> {
-        Table::create(&self.uri, name, batches, params).await
+        let table_uri = self.table_uri(name)?;
+        Table::create(&table_uri, name, batches, params).await
     }
 
     /// Open a table in the database.
@@ -115,7 +115,7 @@ impl Database {
     ///
     /// * A [Table] object.
     pub async fn open_table(&self, name: &str) -> Result<Table> {
-        self.open_table_with_params(name, OpenTableParams::default())
+        self.open_table_with_params(name, &ReadParams::default())
             .await
     }
 
@@ -128,12 +128,9 @@ impl Database {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_table_with_params(
+    pub async fn open_table_with_params(&self, name: &str, params: &ReadParams) -> Result<Table> {
-        &self,
+        let table_uri = self.table_uri(name)?;
-        name: &str,
+        Table::open_with_params(&table_uri, name, params).await
-        params: OpenTableParams,
-    ) -> Result<Table> {
-        Table::open_with_params(&self.uri, name, params).await
     }
 
     /// Drop a table in the database.
@@ -141,10 +138,23 @@ impl Database {
     /// # Arguments
     /// * `name` - The name of the table.
     pub async fn drop_table(&self, name: &str) -> Result<()> {
-        let dir_name = format!("{}/{}.{}", self.uri, name, LANCE_EXTENSION);
+        let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
-        self.object_store.remove_dir_all(dir_name).await?;
+        let full_path = self.base_path.child(dir_name.clone());
+        self.object_store.remove_dir_all(full_path).await?;
         Ok(())
     }
+
+    /// Get the URI of a table in the database.
+    fn table_uri(&self, name: &str) -> Result<String> {
+        let path = Path::new(&self.uri);
+        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+
+        let uri = table_uri
+            .as_path()
+            .to_str()
+            .context(InvalidTableNameSnafu { name })?;
+        Ok(uri.to_string())
+    }
 }
 
 #[cfg(test)]

rust/vectordb/src/index/vector.rs

@@ -35,6 +35,12 @@ pub struct IvfPQIndexBuilder {
 
 impl IvfPQIndexBuilder {
     pub fn new() -> IvfPQIndexBuilder {
+        Default::default()
+    }
+}
+
+impl Default for IvfPQIndexBuilder {
+    fn default() -> Self {
         IvfPQIndexBuilder {
             column: None,
             index_name: None,

rust/vectordb/src/query.rs

@@ -53,7 +53,7 @@ impl Query {
             nprobes: 20,
             refine_factor: None,
             metric_type: None,
-            use_index: false,
+            use_index: true,
             filter: None,
             select: None,
         }

rust/vectordb/src/table.rs

@@ -12,21 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
-use lance::dataset::{Dataset, ReadParams, WriteParams};
+use arrow_schema::SchemaRef;
+use lance::dataset::{Dataset, WriteParams};
 use lance::index::IndexType;
-use snafu::prelude::*;
+use std::path::Path;
 
-use crate::error::{Error, InvalidTableNameSnafu, Result};
+use crate::error::{Error, Result};
 use crate::index::vector::VectorIndexBuilder;
 use crate::query::Query;
 use crate::WriteMode;
 
+pub use lance::dataset::ReadParams;
+
 pub const VECTOR_COLUMN_NAME: &str = "vector";
-pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 /// A table in a LanceDB database.
 #[derive(Debug, Clone)]
@@ -42,24 +43,25 @@ impl std::fmt::Display for Table {
     }
 }
 
-#[derive(Default)]
-pub struct OpenTableParams {
-    pub open_table_params: ReadParams,
-}
-
 impl Table {
     /// Opens an existing Table
     ///
     /// # Arguments
     ///
-    /// * `base_path` - The base path where the table is located
+    /// * `uri` - The uri to a [Table]
-    /// * `name` The Table name
+    /// * `name` - The table name
     ///
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open(base_uri: &str, name: &str) -> Result<Self> {
+    pub async fn open(uri: &str) -> Result<Self> {
-        Self::open_with_params(base_uri, name, OpenTableParams::default()).await
+        let name = Self::get_table_name(uri)?;
+        Self::open_with_params(uri, &name, &ReadParams::default()).await
+    }
+
+    /// Open an Table with a given name.
+    pub async fn open_with_name(uri: &str, name: &str) -> Result<Self> {
+        Self::open_with_params(uri, name, &ReadParams::default()).await
     }
 
     /// Opens an existing Table
@@ -68,25 +70,13 @@ impl Table {
     ///
     /// * `base_path` - The base path where the table is located
     /// * `name` The Table name
-    /// * `params` The [OpenTableParams] to use when opening the table
+    /// * `params` The [ReadParams] to use when opening the table
     ///
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_with_params(
+    pub async fn open_with_params(uri: &str, name: &str, params: &ReadParams) -> Result<Self> {
-        base_uri: &str,
+        let dataset = Dataset::open_with_params(uri, params)
-        name: &str,
-        params: OpenTableParams,
-    ) -> Result<Self> {
-        let path = Path::new(base_uri);
-
-        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
-            .to_str()
-            .context(InvalidTableNameSnafu { name })?;
-
-        let dataset = Dataset::open_with_params(uri, &params.open_table_params)
             .await
             .map_err(|e| match e {
                 lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
@@ -103,31 +93,73 @@ impl Table {
         })
     }
 
+    /// Checkout a specific version of this [`Table`]
+    ///
+    pub async fn checkout(uri: &str, version: u64) -> Result<Self> {
+        let name = Self::get_table_name(uri)?;
+        Self::checkout_with_params(uri, &name, version, &ReadParams::default()).await
+    }
+
+    pub async fn checkout_with_name(uri: &str, name: &str, version: u64) -> Result<Self> {
+        Self::checkout_with_params(uri, name, version, &ReadParams::default()).await
+    }
+
+    pub async fn checkout_with_params(
+        uri: &str,
+        name: &str,
+        version: u64,
+        params: &ReadParams,
+    ) -> Result<Self> {
+        let dataset = Dataset::checkout_with_params(uri, version, params)
+            .await
+            .map_err(|e| match e {
+                lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
+                    name: name.to_string(),
+                },
+                e => Error::Lance {
+                    message: e.to_string(),
+                },
+            })?;
+        Ok(Table {
+            name: name.to_string(),
+            uri: uri.to_string(),
+            dataset: Arc::new(dataset),
+        })
+    }
+
+    fn get_table_name(uri: &str) -> Result<String> {
+        let path = Path::new(uri);
+        let name = path
+            .file_stem()
+            .ok_or(Error::TableNotFound {
+                name: uri.to_string(),
+            })?
+            .to_str()
+            .ok_or(Error::InvalidTableName {
+                name: uri.to_string(),
+            })?;
+        Ok(name.to_string())
+    }
+
     /// Creates a new Table
     ///
     /// # Arguments
     ///
-    /// * `base_path` - The base path where the table is located
+    /// * `uri` - The URI to the table.
     /// * `name` The Table name
-    /// * `batches` RecordBatch to be saved in the database
+    /// * `batches` RecordBatch to be saved in the database.
+    /// * `params` - Write parameters.
     ///
     /// # Returns
     ///
     /// * A [Table] object.
     pub async fn create(
-        base_uri: &str,
+        uri: &str,
         name: &str,
         batches: impl RecordBatchReader + Send + 'static,
         params: Option<WriteParams>,
     ) -> Result<Self> {
-        let base_path = Path::new(base_uri);
+        let dataset = Dataset::write(batches, uri, params)
-        let table_uri = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
-            .to_str()
-            .context(InvalidTableNameSnafu { name })?
-            .to_string();
-        let dataset = Dataset::write(batches, &uri, params)
             .await
             .map_err(|e| match e {
                 lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
@@ -139,11 +171,21 @@ impl Table {
             })?;
         Ok(Table {
             name: name.to_string(),
-            uri,
+            uri: uri.to_string(),
             dataset: Arc::new(dataset),
         })
     }
 
+    /// Schema of this Table.
+    pub fn schema(&self) -> SchemaRef {
+        Arc::new(self.dataset.schema().into())
+    }
+
+    /// Version of this Table
+    pub fn version(&self) -> u64 {
+        self.dataset.version().version
+    }
+
     /// Create index on the table.
     pub async fn create_index(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
         use lance::index::DatasetIndexExt;
@@ -253,14 +295,13 @@ mod tests {
     async fn test_open() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
 
         let batches = make_test_batches();
         Dataset::write(batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
+        let table = Table::open(dataset_path.to_str().unwrap()).await.unwrap();
 
         assert_eq!(table.name, "test")
     }
@@ -269,11 +310,12 @@ mod tests {
     async fn test_open_not_found() {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
-        let table = Table::open(uri, "test").await;
+        let table = Table::open(uri).await;
         assert!(matches!(table.unwrap_err(), Error::TableNotFound { .. }));
     }
 
     #[test]
+    #[cfg(not(windows))]
     fn test_object_store_path() {
         use std::path::Path as StdPath;
         let p = StdPath::new("s3://bucket/path/to/file");
@@ -350,10 +392,7 @@ mod tests {
             ..Default::default()
         };
 
-        table
+        table.add(new_batches, Some(param)).await.unwrap();
-            .add(new_batches, Some(param))
-            .await
-            .unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
         assert_eq!(table.name, "test");
     }
@@ -362,14 +401,14 @@ mod tests {
     async fn test_search() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let uri = dataset_path.to_str().unwrap();
 
         let batches = make_test_batches();
         Dataset::write(batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
+        let table = Table::open(uri).await.unwrap();
 
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = table.search(vector.clone());
@@ -401,7 +440,7 @@ mod tests {
     async fn test_open_table_options() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let uri = dataset_path.to_str().unwrap();
 
         let batches = make_test_batches();
         Dataset::write(batches, dataset_path.to_str().unwrap(), None)
@@ -412,15 +451,12 @@ mod tests {
 
         let mut object_store_params = ObjectStoreParams::default();
         object_store_params.object_store_wrapper = Some(wrapper.clone());
-        let param = OpenTableParams {
+        let param = ReadParams {
-            open_table_params: ReadParams {
+            store_options: Some(object_store_params),
-                store_options: Some(object_store_params),
+            ..Default::default()
-                ..ReadParams::default()
-            },
         };
-
         assert!(!wrapper.called());
-        let _ = Table::open_with_params(uri, "test", param).await.unwrap();
+        let _ = Table::open_with_params(uri, "test", &param).await.unwrap();
         assert!(wrapper.called());
     }