[python] Bump version: 0.1.11 → 0.1.12

`Table::count()` will be linearly slower with more fragments ingested.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.10
+current_version = 0.1.14
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/workflows/docs_test.yml

@@ -81,7 +81,7 @@ jobs:
       run: |
         cd docs/test/node_modules/vectordb 
         npm ci
-        npm run build
+        npm run build-release
         npm run tsc
     - name: Create test files
       run: |

.github/workflows/make-release-commit.yml

@@ -45,6 +45,13 @@ jobs:
       run: |
         pip install bump2version
         bumpversion --verbose ${{ inputs.part }}
+    - name: Update package-lock.json file
+      run: |
+        npm install
+        git add package-lock.json
+        # Add this change to the commit created by bumpversion
+        git commit --amend --no-edit
+      working-directory: node
     - name: Push new version and tag
       if: ${{ inputs.dry_run }} == "false"
       uses: ad-m/github-push-action@master

.github/workflows/node.yml

@@ -67,8 +67,12 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
     - name: Test
       run: npm run test
   macos:
@@ -94,8 +98,12 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
+        # Remove index.node to test with dependency installed
+        rm index.node
     - name: Test
       run: |
         npm run test

.github/workflows/npm-publish.yml

@@ -0,0 +1,172 @@
+name: NPM Publish
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  node:
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - name: Upload Linux Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: node-package
+          path: |
+            node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Install rustup target
+      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+      run: rustup target add aarch64-apple-darwin
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - name: Upload Darwin Artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: darwin-native
+        path: |
+          node/dist/vectordb-darwin*.tgz
+
+  node-linux:
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        libc:
+          - gnu
+          # TODO: re-enable musl once we have refactored to pre-built containers
+          # Right now we have to build node from source which is too expensive.
+          # - musl
+        arch:
+          - x86_64
+          # Building on aarch64 is too slow for now
+          # - aarch64
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Change owner to root (for npm)
+      # The docker container is run as root, so we need the files to be owned by root
+      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+      run: sudo chown -R root:root .
+    - name: Set up QEMU
+      if: ${{ matrix.arch == 'aarch64' }}
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: arm64
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
+      run: |
+        docker run \
+          -v $(pwd):/io -w /io \
+          rust:1.70-bookworm \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
+      run: |
+        docker run --platform linux/arm64/v8 \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+    - name: Upload Linux Artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: linux-native
+        path: |
+          node/dist/vectordb-linux*.tgz
+
+  node-windows:
+    runs-on: windows-2022
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-pc-windows-msvc]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Install npm dependencies
+        run: |
+          cd node
+          npm ci
+      - name: Build Windows native node modules
+        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
+      - name: Upload Windows Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: windows-native
+          path: |
+            node/dist/vectordb-win32*.tgz
+
+  release:
+    needs: [node, node-macos, node-linux]
+    runs-on: ubuntu-latest
+    # Only runs on tags that matches the make-release action
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Display structure of downloaded files
+        run: ls -R
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          registry-url: 'https://registry.npmjs.org'
+      - name: Publish to NPM
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
+        run: |
+          mv */*.tgz .
+          for filename in *.tgz; do
+            npm publish $filename
+          done

.github/workflows/rust.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - Cargo.toml
       - rust/**
       - .github/workflows/rust.yml
 
@@ -65,3 +66,24 @@ jobs:
         run: cargo build --all-features
       - name: Run tests
         run: cargo test --all-features
+  windows:
+    runs-on: windows-2022
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+      - name: Install Protoc v21.12
+        working-directory: C:\
+        run: |
+          New-Item -Path 'C:\protoc' -ItemType Directory
+          Set-Location C:\protoc
+          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
+          7z x protoc.zip
+          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
+        shell: powershell
+      - name: Run tests
+        run: |
+          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
+          cargo build
+          cargo test

.gitignore

@@ -5,6 +5,8 @@
 .DS_Store
 venv
 
+.vscode
+
 rust/target
 rust/Cargo.lock
 

Cargo.toml

@@ -6,9 +6,11 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-lance = "0.5.3"
-arrow-array = "40.0"
-arrow-data = "40.0"
-arrow-schema = "40.0"
-arrow-ipc = "40.0"
+lance = "=0.5.8"
+arrow-array = "42.0"
+arrow-data = "42.0"
+arrow-schema = "42.0"
+arrow-ipc = "42.0"
+half = { "version" = "2.2.1", default-features = false }
 object_store = "0.6.1"
+

ci/build_linux_artifacts.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# TODO: refactor this into a Docker container we can pull
+
+set -e
+
+setup_dependencies() {
+    echo "Installing system dependencies..."
+    if [[ $1 == *musl ]]; then
+        # musllinux
+        apk add openssl-dev
+    else
+        # rust / debian
+        apt update
+        apt install -y libssl-dev protobuf-compiler
+    fi
+}
+
+install_node() {
+    echo "Installing node..."
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+    source "$HOME"/.bashrc
+
+    if [[ $1 == *musl ]]; then
+        # This node version is 15, we need 16 or higher:
+        # apk add nodejs-current npm 
+        # So instead we install from source (nvm doesn't provide binaries for musl):
+        nvm install -s --no-progress 17
+    else
+        nvm install --no-progress 17 # latest that supports glibc 2.17
+    fi
+}
+
+build_node_binary() {
+    echo "Building node library for $1..."
+    pushd node
+
+    npm ci
+    
+    if [[ $1 == *musl ]]; then
+        # This is needed for cargo to allow build cdylibs with musl
+        export RUSTFLAGS="-C target-feature=-crt-static"
+    fi
+
+    # Cargo can run out of memory while pulling dependencies, especially when running
+    # in QEMU. This is a workaround for that.
+    export CARGO_NET_GIT_FETCH_WITH_CLI=true
+
+    # We don't pass in target, since the native target here already matches
+    # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877
+    OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release
+    npm run pack-build
+
+    popd
+}
+
+TARGET=${1:-x86_64-unknown-linux-gnu}
+# Others:
+# aarch64-unknown-linux-gnu
+# x86_64-unknown-linux-musl
+# aarch64-unknown-linux-musl
+
+setup_dependencies $TARGET
+install_node $TARGET
+build_node_binary $TARGET

ci/build_macos_artifacts.sh

@@ -0,0 +1,33 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./ci/build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/ffi/node
+    echo "Building rust library for $1"
+    export RUST_BACKTRACE=1
+    cargo build --release --target $1
+    popd
+}
+
+build_node_binaries() {
+    pushd node
+    echo "Building node library for $1"
+    npm run build-release -- --target $1
+    npm run pack-build -- --target $1
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+echo "Building artifacts for targets: $targets"
+for target in $targets
+    do
+    prebuild_rust $target
+    build_node_binaries $target
+done

ci/build_windows_artifacts.ps1

@@ -0,0 +1,41 @@
+# Builds the Windows artifacts (node binaries).
+# Usage:  .\ci\build_windows_artifacts.ps1 [target]
+# Targets supported:
+# - x86_64-pc-windows-msvc
+# - i686-pc-windows-msvc
+
+function Prebuild-Rust {
+    param (
+        [string]$target
+    )
+
+    # Building here for the sake of easier debugging.
+    Push-Location -Path "rust/ffi/node"
+    Write-Host "Building rust library for $target"
+    $env:RUST_BACKTRACE=1
+    cargo build --release --target $target
+    Pop-Location
+}
+
+function Build-NodeBinaries {
+    param (
+        [string]$target
+    )
+
+    Push-Location -Path "node"
+    Write-Host "Building node library for $target"
+    npm run build-release -- --target $target
+    npm run pack-build -- --target $target
+    Pop-Location
+}
+
+$targets = $args[0]
+if (-not $targets) {
+    $targets = "x86_64-pc-windows-msvc"
+}
+
+Write-Host "Building artifacts for targets: $targets"
+foreach ($target in $targets) {
+    Prebuild-Rust $target
+    Build-NodeBinaries $target
+}

docs/mkdocs.yml

@@ -50,13 +50,19 @@ markdown_extensions:
 - pymdownx.superfences
 - pymdownx.tabbed:
     alternate_style: true
+- md_in_html
 
 nav:
 - Home: index.md
 - Basics: basic.md
 - Embeddings: embedding.md
 - Python full-text search: fts.md
-- Python integrations: integrations.md
+- Python integrations:
+  - Pandas and PyArrow: python/arrow.md
+  - DuckDB: python/duckdb.md
+  - LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+  - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+  - Pydantic: python/pydantic.md
 - Python examples:
   - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
   - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
@@ -65,6 +71,7 @@ nav:
   - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
 - Javascript examples:
   - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+  - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
 - References:
   - Vector Search: search.md
   - SQL filters: sql.md

docs/src/ann_indexes.md

@@ -1,7 +1,7 @@
 # ANN (Approximate Nearest Neighbor) Indexes
 
 You can create an index over your vector data to make search faster.
-Vector indexes are faster but less accurate than exhaustive search.
+Vector indexes are faster but less accurate than exhaustive search (KNN or Flat Search).
 LanceDB provides many parameters to fine-tune the index's size, the speed of queries, and the accuracy of results.
 
 Currently, LanceDB does *not* automatically create the ANN index.
@@ -10,7 +10,18 @@ If you can live with <100ms latency, skipping index creation is a simpler workfl
 
 In the future we will look to automatically create and configure the ANN index.
 
-## Creating an ANN Index
+## Types of Index
+
+Lance can support multiple index types, the most widely used one is `IVF_PQ`.
+
+* `IVF_PQ`: use **Inverted File Index (IVF)** to first divide the dataset into `N` partitions,
+   and then use **Product Quantization** to compress vectors in each partition.
+* `DISKANN` (**Experimental**): organize the vector as a on-disk graph, where the vertices approximately
+   represent the nearest neighbors of each vector.
+
+## Creating an IVF_PQ Index
+
+Lance supports `IVF_PQ` index type by default.
 
 === "Python"
      Creating indexes is done via the [create_index](https://lancedb.github.io/lancedb/python/#lancedb.table.LanceTable.create_index) method.
@@ -45,15 +56,18 @@ In the future we will look to automatically create and configure the ANN index.
      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 256, num_sub_vectors: 96 })
      ```
 
-Since `create_index` has a training step, it can take a few minutes to finish for large tables. You can control the index
-creation by providing the following parameters:
+- **metric** (default: "L2"): The distance metric to use. By default it uses euclidean distance "`L2`".
+We also support "cosine" and "dot" distance as well.
+- **num_partitions** (default: 256): The number of partitions of the index.
+- **num_sub_vectors** (default: 96): The number of sub-vectors (M) that will be created during Product Quantization (PQ).
+For D dimensional vector, it will be divided into `M` of `D/M` sub-vectors, each of which is presented by
+a single PQ code.
+
+<figure markdown>
+  ![IVF PQ](./assets/ivf_pq.png)
+  <figcaption>IVF_PQ index with <code>num_partitions=2, num_sub_vectors=4</code></figcaption>
+</figure>
 
-- **metric** (default: "L2"): The distance metric to use. By default we use euclidean distance. We also support "cosine" distance.
-- **num_partitions** (default: 256): The number of partitions of the index. The number of partitions should be configured so each partition has 3-5K vectors. For example, a table
-with ~1M vectors should use 256 partitions. You can specify arbitrary number of partitions but powers of 2 is most conventional.
-A higher number leads to faster queries, but it makes index generation slower.
-- **num_sub_vectors** (default: 96): The number of subvectors (M) that will be created during Product Quantization (PQ). A larger number makes
-search more accurate, but also makes the index larger and slower to build.
 
 ## Querying an ANN Index
 
@@ -138,3 +152,31 @@ You can select the columns returned by the query using a select clause.
          .select(["id"])
          .execute()
      ```
+
+## FAQ
+
+### When is it necessary to create an ANN vector index.
+
+`LanceDB` has manually tuned SIMD code for computing vector distances.
+In our benchmarks, computing 100K pairs of 1K dimension vectors only take less than 20ms.
+For small dataset (<100K rows) or the applications which can accept 100ms latency, vector indices are usually not necessary.
+
+For large-scale or higher dimension vectors, it is beneficial to create vector index.
+
+### How big is my index, and how many memory will it take.
+
+In LanceDB, all vector indices are disk-based, meaning that when responding to a vector query, only the relevant pages from the index file are loaded from disk and cached in memory. Additionally, each sub-vector is usually encoded into 1 byte PQ code.
+
+For example, with a 1024-dimension dataset, if we choose `num_sub_vectors=64`, each sub-vector has `1024 / 64 = 16` float32 numbers.
+Product quantization can lead to approximately `16 * sizeof(float32) / 1 = 64` times of space reduction.
+
+### How to choose `num_partitions` and `num_sub_vectors` for `IVF_PQ` index.
+
+`num_partitions` is used to decide how many partitions the first level `IVF` index uses.
+Higher number of partitions could lead to more efficient I/O during queries and better accuracy, but it takes much more time to train.
+On `SIFT-1M` dataset, our benchmark shows that keeping each partition 1K-4K rows lead to a good latency / recall.
+
+`num_sub_vectors` decides how many Product Quantization code to generate on each vector. Because
+Product Quantization is a lossy compression of the original vector, the more `num_sub_vectors` usually results to
+less space distortion, and thus yield better accuracy. However, similarly, more `num_sub_vectors` causes heavier I/O and
+more PQ computation, thus, higher latency. `dimension / num_sub_vectors` should be aligned with 8 for better SIMD efficiency.

docs/src/basic.md

@@ -122,6 +122,35 @@ After a table has been created, you can always add more data to it using
               {vector: [9.5, 56.2], item: "buzz", price: 200.0}])
       ```
 
+## How to delete rows from a table
+
+Use the `delete()` method on tables to delete rows from a table. To choose
+which rows to delete, provide a filter that matches on the metadata columns.
+This can delete any number of rows that match the filter.
+
+=== "Python"
+      ```python
+      tbl.delete('item = "fizz"')
+      ```
+
+=== "Javascript"
+      ```javascript
+      await tbl.delete('item = "fizz"')
+      ```
+
+The deletion predicate is a SQL expression that supports the same expressions
+as the `where()` clause on a search. They can be as simple or complex as needed.
+To see what expressions are supported, see the [SQL filters](sql.md) section.
+
+
+=== "Python"
+
+      Read more: [lancedb.table.Table.delete][]
+
+=== "Javascript"
+
+      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+
 ## How to search for (approximate) nearest neighbors
 
 Once you've embedded the query, you can find its nearest neighbors using the following code:

docs/src/embedding.md

@@ -46,7 +46,7 @@ You can also use an external API like OpenAI to generate embeddings
 
         def embed_func(c):
             rs = openai.Embedding.create(input=c, engine="text-embedding-ada-002")
-        return [record["embedding"] for record in rs["data"]]
+            return [record["embedding"] for record in rs["data"]]
       ```
 
 === "Javascript"
@@ -126,7 +126,7 @@ belong in the same latent space and your results will be nonsensical.
 === "Javascript"
      ```javascript
       const results = await table
-        .search('What's the best pizza topping?')
+        .search("What's the best pizza topping?")
         .limit(10)
         .execute()
      ```

docs/src/examples/transformerjs_embedding_search_nodejs.md

@@ -0,0 +1,121 @@
+# Vector embedding search using TransformersJS
+
+## Embed and query data from LacneDB using TransformersJS
+
+<img id="splash" width="400" alt="transformersjs" src="https://github.com/lancedb/lancedb/assets/43097991/88a31e30-3d6f-4eef-9216-4b7c688f1b4f">
+
+This example shows how to use the [transformers.js](https://github.com/xenova/transformers.js) library to perform vector embedding search using LanceDB's Javascript API.
+
+
+### Setting up
+First, install the dependencies:
+```bash
+npm install vectordb
+npm i @xenova/transformers
+```
+
+We will also be using the [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) model to make it compatible with Transformers.js
+
+Within our `index.js` file we will import the necessary libraries and define our model and database:
+
+```javascript
+const lancedb = require('vectordb')
+const { pipeline } = await import('@xenova/transformers')
+const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+```
+
+### Creating the embedding function
+
+Next, we will create a function that will take in a string and return the vector embedding of that string. We will use the `pipe` function we defined earlier to get the vector embedding of the string.
+
+```javascript
+// Define the function. `sourceColumn` is required for LanceDB to know
+// which column to use as input.
+const embed_fun = {}
+embed_fun.sourceColumn = 'text'
+embed_fun.embed = async function (batch) {
+    let result = []
+    // Given a batch of strings, we will use the `pipe` function to get
+    // the vector embedding of each string.
+    for (let text of batch) {
+        // 'mean' pooling and normalizing allows the embeddings to share the
+        // same length.
+        const res = await pipe(text, { pooling: 'mean', normalize: true })
+        result.push(Array.from(res['data']))
+    }
+    return (result)
+}
+```
+
+### Creating the database
+
+Now, we will create the LanceDB database and add the embedding function we defined earlier.
+
+```javascript
+// Link a folder and create a table with data
+const db = await lancedb.connect('data/sample-lancedb')
+
+// You can also import any other data, but make sure that you have a column
+// for the embedding function to use.
+const data = [
+    { id: 1, text: 'Cherry', type: 'fruit' },
+    { id: 2, text: 'Carrot', type: 'vegetable' },
+    { id: 3, text: 'Potato', type: 'vegetable' },
+    { id: 4, text: 'Apple', type: 'fruit' },
+    { id: 5, text: 'Banana', type: 'fruit' }
+]
+
+// Create the table with the embedding function
+const table = await db.createTable('food_table', data, "create", embed_fun)
+```
+
+### Performing the search
+
+Now, we can perform the search using the `search` function. LanceDB automatically uses the embedding function we defined earlier to get the vector embedding of the query string.
+
+```javascript
+// Query the table
+const results = await table
+    .search("a sweet fruit to eat")
+    .metricType("cosine")
+    .limit(2)
+    .execute()
+console.log(results.map(r => r.text))
+```
+```bash
+[ 'Banana', 'Cherry' ]
+```
+
+Output of `results`:
+```bash
+[
+  {
+    vector: Float32Array(384) [
+      -0.057455405592918396,
+      0.03617725893855095,
+      -0.0367760956287384,
+      ... 381 more items
+    ],
+    id: 5,
+    text: 'Banana',
+    type: 'fruit',
+    score: 0.4919965863227844
+  },
+  {
+    vector: Float32Array(384) [
+      0.0009714411571621895,
+      0.008223623037338257,
+      0.009571489877998829,
+      ... 381 more items
+    ],
+    id: 1,
+    text: 'Cherry',
+    type: 'fruit',
+    score: 0.5540297031402588
+  }
+]
+```
+
+### Wrapping it up
+
+In this example, we showed how to use the `transformers.js` library to perform vector embedding search using LanceDB's Javascript API. You can find the full code for this example on [Github](https://github.com/lancedb/lancedb/blob/main/node/examples/js-transformers/index.js)!

docs/src/index.md

@@ -46,7 +46,7 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 
       const uri = "data/sample-lancedb";
       const db = await lancedb.connect(uri);
-      const table = await db.createTable("my_table", 
+      const table = await db.createTable("my_table",
             [{ id: 1, vector: [3.1, 4.1], item: "foo", price: 10.0 },
             { id: 2, vector: [5.9, 26.5], item: "bar", price: 20.0 }])
       const results = await table.search([100, 100]).limit(2).execute();
@@ -67,6 +67,6 @@ LanceDB's core is written in Rust 🦀 and is built using <a href="https://githu
 * [`Embedding Functions`](embedding.md) - functions for working with embeddings.
 * [`Indexing`](ann_indexes.md) - create vector indexes to speed up queries.
 * [`Full text search`](fts.md) - [EXPERIMENTAL] full-text search API
-* [`Ecosystem Integrations`](integrations.md) - integrating LanceDB with python data tooling ecosystem.
+* [`Ecosystem Integrations`](python/integration.md) - integrating LanceDB with python data tooling ecosystem.
 * [`Python API Reference`](python/python.md) - detailed documentation for the LanceDB Python SDK.
 * [`Node API Reference`](javascript/modules.md) - detailed documentation for the LanceDB Python SDK.

docs/src/integrations.md

@@ -1,116 +0,0 @@
-# Integrations
-
-Built on top of Apache Arrow, `LanceDB` is easy to integrate with the Python ecosystem, including Pandas, PyArrow and DuckDB.
-
-## Pandas and PyArrow
-
-First, we need to connect to a `LanceDB` database.
-
-```py
-
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-```
-
-And write a `Pandas DataFrame` to LanceDB directly.
-
-```py
-import pandas as pd
-
-data = pd.DataFrame({
-    "vector": [[3.1, 4.1], [5.9, 26.5]],
-    "item": ["foo", "bar"],
-    "price": [10.0, 20.0]
-})
-table = db.create_table("pd_table", data=data)
-```
-
-You will find detailed instructions of creating dataset and index in [Basic Operations](basic.md) and [Indexing](ann_indexes.md)
-sections.
-
-
-We can now perform similarity searches via `LanceDB`.
-
-```py
-# Open the table previously created.
-table = db.open_table("pd_table")
-
-query_vector = [100, 100]
-# Pandas DataFrame
-df = table.search(query_vector).limit(1).to_df()
-print(df)
-```
-
-```
-    vector     item  price        score
-0  [5.9, 26.5]  bar   20.0  14257.05957
-```
-
-If you have a simple filter, it's faster to provide a where clause to `LanceDB`'s search query.
-If you have more complex criteria, you can always apply the filter to the resulting pandas `DataFrame` from the search query.
-
-```python
-
-# Apply the filter via LanceDB
-results = table.search([100, 100]).where("price < 15").to_df()
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-
-# Apply the filter via Pandas
-df = results = table.search([100, 100]).to_df()
-results = df[df.price < 15]
-assert len(results) == 1
-assert results["item"].iloc[0] == "foo"
-```
-
-## DuckDB
-
-`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
-
-Let us start with installing `duckdb` and `lancedb`.
-
-```shell
-pip install duckdb lancedb
-```
-
-We will re-use the dataset created previously
-
-```python
-import lancedb
-
-db = lancedb.connect("data/sample-lancedb")
-table = db.open_table("pd_table")
-arrow_table = table.to_arrow()
-```
-
-`DuckDB` can directly query the `arrow_table`:
-
-```python
-import duckdb 
-
-duckdb.query("SELECT * FROM arrow_table")
-```
-
-```
-┌─────────────┬─────────┬────────┐
-│   vector    │  item   │ price  │
-│   float[]   │ varchar │ double │
-├─────────────┼─────────┼────────┤
-│ [3.1, 4.1]  │ foo     │   10.0 │
-│ [5.9, 26.5] │ bar     │   20.0 │
-└─────────────┴─────────┴────────┘
-```
-```python
-duckdb.query("SELECT mean(price) FROM arrow_table")
-```
-
-```
-Out[16]:
-┌─────────────┐
-│ mean(price) │
-│   double    │
-├─────────────┤
-│        15.0 │
-└─────────────┘
-```

docs/src/javascript/README.md

@@ -10,6 +10,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
@@ -28,12 +32,34 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-Run the tests with
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
 
 ```bash
 npm test
 ```
 
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript
+
+```bash
+npm run tsc
+```
+
+### Fix lints
+
 To run the linter and have it automatically fix all errors
 
 ```bash

docs/src/javascript/classes/LocalConnection.md

@@ -17,7 +17,7 @@ A connection to a LanceDB database.
 ### Properties
 
 - [\_db](LocalConnection.md#_db)
-- [\_uri](LocalConnection.md#_uri)
+- [\_options](LocalConnection.md#_options)
 
 ### Accessors
 
@@ -35,18 +35,18 @@ A connection to a LanceDB database.
 
 ### constructor
 
-• **new LocalConnection**(`db`, `uri`)
+• **new LocalConnection**(`db`, `options`)
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
 | `db` | `any` |
-| `uri` | `string` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
 
 #### Defined in
 
-[index.ts:132](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L132)
+[index.ts:184](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L184)
 
 ## Properties
 
@@ -56,17 +56,17 @@ A connection to a LanceDB database.
 
 #### Defined in
 
-[index.ts:130](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L130)
+[index.ts:182](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L182)
 
 ___
 
-### \_uri
+### \_options
 
-• `Private` `Readonly` **\_uri**: `string`
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
 
 #### Defined in
 
-[index.ts:129](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L129)
+[index.ts:181](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L181)
 
 ## Accessors
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:137](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L137)
+[index.ts:189](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L189)
 
 ## Methods
 
@@ -112,7 +112,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:177](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L177)
+[index.ts:230](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L230)
 
 ▸ **createTable**(`name`, `data`, `mode`): `Promise`<[`Table`](../interfaces/Table.md)<`number`[]\>\>
 
@@ -134,7 +134,7 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:178](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L178)
+[index.ts:231](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L231)
 
 ▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -165,7 +165,36 @@ Connection.createTable
 
 #### Defined in
 
-[index.ts:188](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L188)
+[index.ts:241](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L241)
+
+▸ **createTable**<`T`\>(`name`, `data`, `mode`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `data` | `Record`<`string`, `unknown`\>[] |
+| `mode` | [`WriteMode`](../enums/WriteMode.md) |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.createTable
+
+#### Defined in
+
+[index.ts:242](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L242)
 
 ___
 
@@ -190,7 +219,7 @@ ___
 
 #### Defined in
 
-[index.ts:201](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L201)
+[index.ts:266](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L266)
 
 ___
 
@@ -216,7 +245,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:211](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L211)
+[index.ts:276](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L276)
 
 ___
 
@@ -242,7 +271,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:153](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L153)
+[index.ts:205](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L205)
 
 ▸ **openTable**<`T`\>(`name`, `embeddings`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
 
@@ -271,7 +300,34 @@ Connection.openTable
 
 #### Defined in
 
-[index.ts:160](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L160)
+[index.ts:212](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L212)
+
+▸ **openTable**<`T`\>(`name`, `embeddings?`): `Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Type parameters
+
+| Name |
+| :------ |
+| `T` |
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `embeddings?` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> |
+
+#### Returns
+
+`Promise`<[`Table`](../interfaces/Table.md)<`T`\>\>
+
+#### Implementation of
+
+Connection.openTable
+
+#### Defined in
+
+[index.ts:213](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L213)
 
 ___
 
@@ -291,4 +347,4 @@ Get the names of all tables in the database.
 
 #### Defined in
 
-[index.ts:144](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L144)
+[index.ts:196](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L196)

docs/src/javascript/classes/LocalTable.md

@@ -24,6 +24,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 - [\_embeddings](LocalTable.md#_embeddings)
 - [\_name](LocalTable.md#_name)
+- [\_options](LocalTable.md#_options)
 - [\_tbl](LocalTable.md#_tbl)
 
 ### Accessors
@@ -43,7 +44,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 ### constructor
 
-• **new LocalTable**<`T`\>(`tbl`, `name`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`)
 
 #### Type parameters
 
@@ -57,12 +58,13 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ |
 | `tbl` | `any` |
 | `name` | `string` |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |
 
 #### Defined in
 
-[index.ts:221](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L221)
+[index.ts:287](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L287)
 
-• **new LocalTable**<`T`\>(`tbl`, `name`, `embeddings`)
+• **new LocalTable**<`T`\>(`tbl`, `name`, `options`, `embeddings`)
 
 #### Type parameters
 
@@ -76,11 +78,12 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 | :------ | :------ | :------ |
 | `tbl` | `any` |  |
 | `name` | `string` |  |
+| `options` | [`ConnectionOptions`](../interfaces/ConnectionOptions.md) |  |
 | `embeddings` | [`EmbeddingFunction`](../interfaces/EmbeddingFunction.md)<`T`\> | An embedding function to use when interacting with this table |
 
 #### Defined in
 
-[index.ts:227](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L227)
+[index.ts:294](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L294)
 
 ## Properties
 
@@ -90,7 +93,7 @@ A LanceDB Table is the collection of Records. Each Record has one or more vector
 
 #### Defined in
 
-[index.ts:219](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L219)
+[index.ts:284](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L284)
 
 ___
 
@@ -100,7 +103,17 @@ ___
 
 #### Defined in
 
-[index.ts:218](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L218)
+[index.ts:283](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L283)
+
+___
+
+### \_options
+
+• `Private` `Readonly` **\_options**: [`ConnectionOptions`](../interfaces/ConnectionOptions.md)
+
+#### Defined in
+
+[index.ts:285](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L285)
 
 ___
 
@@ -110,7 +123,7 @@ ___
 
 #### Defined in
 
-[index.ts:217](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L217)
+[index.ts:282](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L282)
 
 ## Accessors
 
@@ -128,7 +141,7 @@ ___
 
 #### Defined in
 
-[index.ts:234](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L234)
+[index.ts:302](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L302)
 
 ## Methods
 
@@ -156,7 +169,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:252](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L252)
+[index.ts:320](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L320)
 
 ___
 
@@ -176,7 +189,7 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:278](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L278)
+[index.ts:362](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L362)
 
 ___
 
@@ -194,7 +207,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+| `indexParams` | [`IvfPQIndexConfig`](../interfaces/IvfPQIndexConfig.md) | The parameters of this Index, |
 
 #### Returns
 
@@ -206,7 +219,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:271](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L271)
+[index.ts:355](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L355)
 
 ___
 
@@ -232,7 +245,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:287](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L287)
+[index.ts:371](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L371)
 
 ___
 
@@ -260,7 +273,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:262](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L262)
+[index.ts:338](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L338)
 
 ___
 
@@ -286,4 +299,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:242](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L242)
+[index.ts:310](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L310)

docs/src/javascript/classes/OpenAIEmbeddingFunction.md

@@ -40,7 +40,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L21)
+[embedding/openai.ts:21](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L21)
 
 ## Properties
 
@@ -50,7 +50,7 @@ An embedding function that automatically creates vector representation for a giv
 
 #### Defined in
 
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L19)
+[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L19)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L18)
+[embedding/openai.ts:18](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L18)
 
 ___
 
@@ -76,7 +76,7 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L50)
+[embedding/openai.ts:50](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L50)
 
 ## Methods
 
@@ -102,4 +102,4 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/openai.ts#L38)
+[embedding/openai.ts:38](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/openai.ts#L38)

docs/src/javascript/classes/Query.md

@@ -62,7 +62,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:362](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L362)
+[index.ts:448](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L448)
 
 ## Properties
 
@@ -72,7 +72,7 @@ A builder for nearest neighbor queries for LanceDB.
 
 #### Defined in
 
-[index.ts:360](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L360)
+[index.ts:446](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L446)
 
 ___
 
@@ -82,7 +82,7 @@ ___
 
 #### Defined in
 
-[index.ts:358](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L358)
+[index.ts:444](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L444)
 
 ___
 
@@ -92,7 +92,7 @@ ___
 
 #### Defined in
 
-[index.ts:354](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L354)
+[index.ts:440](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L440)
 
 ___
 
@@ -102,7 +102,7 @@ ___
 
 #### Defined in
 
-[index.ts:359](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L359)
+[index.ts:445](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L445)
 
 ___
 
@@ -112,7 +112,7 @@ ___
 
 #### Defined in
 
-[index.ts:356](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L356)
+[index.ts:442](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L442)
 
 ___
 
@@ -122,7 +122,7 @@ ___
 
 #### Defined in
 
-[index.ts:352](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L352)
+[index.ts:438](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L438)
 
 ___
 
@@ -132,7 +132,7 @@ ___
 
 #### Defined in
 
-[index.ts:353](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L353)
+[index.ts:439](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L439)
 
 ___
 
@@ -142,7 +142,7 @@ ___
 
 #### Defined in
 
-[index.ts:355](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L355)
+[index.ts:441](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L441)
 
 ___
 
@@ -152,7 +152,7 @@ ___
 
 #### Defined in
 
-[index.ts:357](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L357)
+[index.ts:443](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L443)
 
 ___
 
@@ -162,7 +162,7 @@ ___
 
 #### Defined in
 
-[index.ts:351](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L351)
+[index.ts:437](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L437)
 
 ___
 
@@ -188,7 +188,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:410](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L410)
+[index.ts:496](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L496)
 
 ## Methods
 
@@ -210,7 +210,7 @@ Execute the query and return the results as an Array of Objects
 
 #### Defined in
 
-[index.ts:433](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L433)
+[index.ts:519](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L519)
 
 ___
 
@@ -232,7 +232,7 @@ A filter statement to be applied to this query.
 
 #### Defined in
 
-[index.ts:405](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L405)
+[index.ts:491](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L491)
 
 ___
 
@@ -254,7 +254,7 @@ Sets the number of results that will be returned
 
 #### Defined in
 
-[index.ts:378](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L378)
+[index.ts:464](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L464)
 
 ___
 
@@ -280,7 +280,7 @@ MetricType for the different options
 
 #### Defined in
 
-[index.ts:425](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L425)
+[index.ts:511](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L511)
 
 ___
 
@@ -302,7 +302,7 @@ The number of probes used. A higher number makes search more accurate but also s
 
 #### Defined in
 
-[index.ts:396](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L396)
+[index.ts:482](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L482)
 
 ___
 
@@ -324,7 +324,7 @@ Refine the results by reading extra elements and re-ranking them in memory.
 
 #### Defined in
 
-[index.ts:387](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L387)
+[index.ts:473](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L473)
 
 ___
 
@@ -346,4 +346,4 @@ Return only the specified columns.
 
 #### Defined in
 
-[index.ts:416](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L416)
+[index.ts:502](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L502)

docs/src/javascript/enums/MetricType.md

@@ -22,7 +22,7 @@ Cosine distance
 
 #### Defined in
 
-[index.ts:481](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L481)
+[index.ts:567](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L567)
 
 ___
 
@@ -34,7 +34,7 @@ Dot product
 
 #### Defined in
 
-[index.ts:486](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L486)
+[index.ts:572](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L572)
 
 ___
 
@@ -46,4 +46,4 @@ Euclidean distance
 
 #### Defined in
 
-[index.ts:476](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L476)
+[index.ts:562](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L562)

docs/src/javascript/enums/WriteMode.md

@@ -22,7 +22,7 @@ Append new data to the table.
 
 #### Defined in
 
-[index.ts:466](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L466)
+[index.ts:552](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L552)
 
 ___
 
@@ -34,7 +34,7 @@ Create a new [Table](../interfaces/Table.md).
 
 #### Defined in
 
-[index.ts:462](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L462)
+[index.ts:548](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L548)
 
 ___
 
@@ -46,4 +46,4 @@ Overwrite the existing [Table](../interfaces/Table.md) if presented.
 
 #### Defined in
 
-[index.ts:464](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L464)
+[index.ts:550](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L550)

docs/src/javascript/interfaces/AwsCredentials.md

@@ -0,0 +1,41 @@
+[vectordb](../README.md) / [Exports](../modules.md) / AwsCredentials
+
+# Interface: AwsCredentials
+
+## Table of contents
+
+### Properties
+
+- [accessKeyId](AwsCredentials.md#accesskeyid)
+- [secretKey](AwsCredentials.md#secretkey)
+- [sessionToken](AwsCredentials.md#sessiontoken)
+
+## Properties
+
+### accessKeyId
+
+• **accessKeyId**: `string`
+
+#### Defined in
+
+[index.ts:31](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L31)
+
+___
+
+### secretKey
+
+• **secretKey**: `string`
+
+#### Defined in
+
+[index.ts:33](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L33)
+
+___
+
+### sessionToken
+
+• `Optional` **sessionToken**: `string`
+
+#### Defined in
+
+[index.ts:35](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L35)

docs/src/javascript/interfaces/Connection.md

@@ -32,7 +32,7 @@ Connection could be local against filesystem or remote against a server.
 
 #### Defined in
 
-[index.ts:45](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L45)
+[index.ts:70](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L70)
 
 ## Methods
 
@@ -63,7 +63,7 @@ Creates a new Table and initialize it with new data.
 
 #### Defined in
 
-[index.ts:65](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L65)
+[index.ts:90](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L90)
 
 ___
 
@@ -84,7 +84,7 @@ ___
 
 #### Defined in
 
-[index.ts:67](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L67)
+[index.ts:92](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L92)
 
 ___
 
@@ -106,7 +106,7 @@ Drop an existing table.
 
 #### Defined in
 
-[index.ts:73](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L73)
+[index.ts:98](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L98)
 
 ___
 
@@ -135,7 +135,7 @@ Open a table in the database.
 
 #### Defined in
 
-[index.ts:55](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L55)
+[index.ts:80](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L80)
 
 ___
 
@@ -149,4 +149,4 @@ ___
 
 #### Defined in
 
-[index.ts:47](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L47)
+[index.ts:72](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L72)

docs/src/javascript/interfaces/ConnectionOptions.md

@@ -0,0 +1,30 @@
+[vectordb](../README.md) / [Exports](../modules.md) / ConnectionOptions
+
+# Interface: ConnectionOptions
+
+## Table of contents
+
+### Properties
+
+- [awsCredentials](ConnectionOptions.md#awscredentials)
+- [uri](ConnectionOptions.md#uri)
+
+## Properties
+
+### awsCredentials
+
+• `Optional` **awsCredentials**: [`AwsCredentials`](AwsCredentials.md)
+
+#### Defined in
+
+[index.ts:40](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L40)
+
+___
+
+### uri
+
+• **uri**: `string`
+
+#### Defined in
+
+[index.ts:39](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L39)

docs/src/javascript/interfaces/EmbeddingFunction.md

@@ -45,7 +45,7 @@ Creates a vector representation for the given values.
 
 #### Defined in
 
-[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L27)
+[embedding/embedding_function.ts:27](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L27)
 
 ___
 
@@ -57,4 +57,4 @@ The name of the column that will be used as input for the Embedding Function.
 
 #### Defined in
 
-[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/7247834/node/src/embedding/embedding_function.ts#L22)
+[embedding/embedding_function.ts:22](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/embedding/embedding_function.ts#L22)

docs/src/javascript/interfaces/IvfPQIndexConfig.md

@@ -0,0 +1,149 @@
+[vectordb](../README.md) / [Exports](../modules.md) / IvfPQIndexConfig
+
+# Interface: IvfPQIndexConfig
+
+## Table of contents
+
+### Properties
+
+- [column](IvfPQIndexConfig.md#column)
+- [index\_name](IvfPQIndexConfig.md#index_name)
+- [max\_iters](IvfPQIndexConfig.md#max_iters)
+- [max\_opq\_iters](IvfPQIndexConfig.md#max_opq_iters)
+- [metric\_type](IvfPQIndexConfig.md#metric_type)
+- [num\_bits](IvfPQIndexConfig.md#num_bits)
+- [num\_partitions](IvfPQIndexConfig.md#num_partitions)
+- [num\_sub\_vectors](IvfPQIndexConfig.md#num_sub_vectors)
+- [replace](IvfPQIndexConfig.md#replace)
+- [type](IvfPQIndexConfig.md#type)
+- [use\_opq](IvfPQIndexConfig.md#use_opq)
+
+## Properties
+
+### column
+
+• `Optional` **column**: `string`
+
+The column to be indexed
+
+#### Defined in
+
+[index.ts:382](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L382)
+
+___
+
+### index\_name
+
+• `Optional` **index\_name**: `string`
+
+A unique name for the index
+
+#### Defined in
+
+[index.ts:387](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L387)
+
+___
+
+### max\_iters
+
+• `Optional` **max\_iters**: `number`
+
+The max number of iterations for kmeans training.
+
+#### Defined in
+
+[index.ts:402](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L402)
+
+___
+
+### max\_opq\_iters
+
+• `Optional` **max\_opq\_iters**: `number`
+
+Max number of iterations to train OPQ, if `use_opq` is true.
+
+#### Defined in
+
+[index.ts:421](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L421)
+
+___
+
+### metric\_type
+
+• `Optional` **metric\_type**: [`MetricType`](../enums/MetricType.md)
+
+Metric type, L2 or Cosine
+
+#### Defined in
+
+[index.ts:392](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L392)
+
+___
+
+### num\_bits
+
+• `Optional` **num\_bits**: `number`
+
+The number of bits to present one PQ centroid.
+
+#### Defined in
+
+[index.ts:416](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L416)
+
+___
+
+### num\_partitions
+
+• `Optional` **num\_partitions**: `number`
+
+The number of partitions this index
+
+#### Defined in
+
+[index.ts:397](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L397)
+
+___
+
+### num\_sub\_vectors
+
+• `Optional` **num\_sub\_vectors**: `number`
+
+Number of subvectors to build PQ code
+
+#### Defined in
+
+[index.ts:412](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L412)
+
+___
+
+### replace
+
+• `Optional` **replace**: `boolean`
+
+Replace an existing index with the same name if it exists.
+
+#### Defined in
+
+[index.ts:426](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L426)
+
+___
+
+### type
+
+• **type**: ``"ivf_pq"``
+
+#### Defined in
+
+[index.ts:428](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L428)
+
+___
+
+### use\_opq
+
+• `Optional` **use\_opq**: `boolean`
+
+Train as optimized product quantization.
+
+#### Defined in
+
+[index.ts:407](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L407)

docs/src/javascript/interfaces/Table.md

@@ -52,7 +52,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:95](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L95)
+[index.ts:120](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L120)
 
 ___
 
@@ -72,13 +72,13 @@ Returns the number of rows in this table.
 
 #### Defined in
 
-[index.ts:115](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L115)
+[index.ts:140](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L140)
 
 ___
 
 ### createIndex
 
-• **createIndex**: (`indexParams`: `IvfPQIndexConfig`) => `Promise`<`any`\>
+• **createIndex**: (`indexParams`: [`IvfPQIndexConfig`](IvfPQIndexConfig.md)) => `Promise`<`any`\>
 
 #### Type declaration
 
@@ -94,7 +94,7 @@ VectorIndexParams.
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `indexParams` | `IvfPQIndexConfig` | The parameters of this Index, |
+| `indexParams` | [`IvfPQIndexConfig`](IvfPQIndexConfig.md) | The parameters of this Index, |
 
 ##### Returns
 
@@ -102,7 +102,7 @@ VectorIndexParams.
 
 #### Defined in
 
-[index.ts:110](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L110)
+[index.ts:135](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L135)
 
 ___
 
@@ -116,11 +116,37 @@ ___
 
 Delete rows from this table.
 
+This can be used to delete a single row, many rows, all rows, or
+sometimes no rows (if your predicate matches nothing).
+
+**`Examples`**
+
+```ts
+const con = await lancedb.connect("./.lancedb")
+const data = [
+   {id: 1, vector: [1, 2]},
+   {id: 2, vector: [3, 4]},
+   {id: 3, vector: [5, 6]},
+];
+const tbl = await con.createTable("my_table", data)
+await tbl.delete("id = 2")
+await tbl.countRows() // Returns 2
+```
+
+If you have a list of values to delete, you can combine them into a
+stringified list and use the `IN` operator:
+
+```ts
+const to_remove = [1, 5];
+await tbl.delete(`id IN (${to_remove.join(",")})`)
+await tbl.countRows() // Returns 1
+```
+
 ##### Parameters
 
 | Name | Type | Description |
 | :------ | :------ | :------ |
-| `filter` | `string` | A filter in the same format used by a sql WHERE clause. |
+| `filter` | `string` | A filter in the same format used by a sql WHERE clause. The filter must not be empty. |
 
 ##### Returns
 
@@ -128,7 +154,7 @@ Delete rows from this table.
 
 #### Defined in
 
-[index.ts:122](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L122)
+[index.ts:174](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L174)
 
 ___
 
@@ -138,7 +164,7 @@ ___
 
 #### Defined in
 
-[index.ts:81](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L81)
+[index.ts:106](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L106)
 
 ___
 
@@ -166,7 +192,7 @@ The number of rows added to the table
 
 #### Defined in
 
-[index.ts:103](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L103)
+[index.ts:128](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L128)
 
 ___
 
@@ -192,4 +218,4 @@ Creates a search query to find the nearest neighbors of the given search term
 
 #### Defined in
 
-[index.ts:87](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L87)
+[index.ts:112](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L112)

docs/src/javascript/modules.md

@@ -18,8 +18,11 @@
 
 ### Interfaces
 
+- [AwsCredentials](interfaces/AwsCredentials.md)
 - [Connection](interfaces/Connection.md)
+- [ConnectionOptions](interfaces/ConnectionOptions.md)
 - [EmbeddingFunction](interfaces/EmbeddingFunction.md)
+- [IvfPQIndexConfig](interfaces/IvfPQIndexConfig.md)
 - [Table](interfaces/Table.md)
 
 ### Type Aliases
@@ -34,11 +37,11 @@
 
 ### VectorIndexParams
 
-Ƭ **VectorIndexParams**: `IvfPQIndexConfig`
+Ƭ **VectorIndexParams**: [`IvfPQIndexConfig`](interfaces/IvfPQIndexConfig.md)
 
 #### Defined in
 
-[index.ts:345](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L345)
+[index.ts:431](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L431)
 
 ## Functions
 
@@ -60,4 +63,20 @@ Connect to a LanceDB instance at the given URI
 
 #### Defined in
 
-[index.ts:34](https://github.com/lancedb/lancedb/blob/7247834/node/src/index.ts#L34)
+[index.ts:47](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L47)
+
+▸ **connect**(`opts`): `Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `opts` | `Partial`<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> |
+
+#### Returns
+
+`Promise`<[`Connection`](interfaces/Connection.md)\>
+
+#### Defined in
+
+[index.ts:48](https://github.com/lancedb/lancedb/blob/b1eeb90/node/src/index.ts#L48)

docs/src/python/arrow.md

@@ -0,0 +1,101 @@
+# Pandas and PyArrow
+
+
+Built on top of [Apache Arrow](https://arrow.apache.org/),
+`LanceDB` is easy to integrate with the Python ecosystem, including [Pandas](https://pandas.pydata.org/)
+and PyArrow.
+
+## Create dataset
+
+First, we need to connect to a `LanceDB` database.
+
+```py
+
+import lancedb
+
+db = lancedb.connect("data/sample-lancedb")
+```
+
+Afterwards, we write a `Pandas DataFrame` to LanceDB directly.
+
+```py
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+```
+
+Similar to [`pyarrow.write_dataset()`](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.write_dataset.html),
+[db.create_table()](../python/#lancedb.db.DBConnection.create_table) accepts a wide-range of forms of data.
+
+For example, if you have a dataset that is larger than memory size, you can create table with `Iterator[pyarrow.RecordBatch]`,
+to lazily generate data:
+
+```py
+
+from typing import Iterable
+import pyarrow as pa
+import lancedb
+
+def make_batches() -> Iterable[pa.RecordBatch]:
+    for i in range(5):
+        yield pa.RecordBatch.from_arrays(
+            [
+                pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                pa.array(["foo", "bar"]),
+                pa.array([10.0, 20.0]),
+            ],
+            ["vector", "item", "price"])
+
+schema=pa.schema([
+    pa.field("vector", pa.list_(pa.float32())),
+    pa.field("item", pa.utf8()),
+    pa.field("price", pa.float32()),
+])
+
+table = db.create_table("iterable_table", data=make_batches(), schema=schema)
+```
+
+You will find detailed instructions of creating dataset in
+[Basic Operations](../basic.md) and [API](../python/#lancedb.db.DBConnection.create_table)
+sections.
+
+## Vector Search
+
+We can now perform similarity search via `LanceDB` Python API.
+
+```py
+# Open the table previously created.
+table = db.open_table("pd_table")
+
+query_vector = [100, 100]
+# Pandas DataFrame
+df = table.search(query_vector).limit(1).to_df()
+print(df)
+```
+
+```
+    vector     item  price        score
+0  [5.9, 26.5]  bar   20.0  14257.05957
+```
+
+If you have a simple filter, it's faster to provide a `where clause` to `LanceDB`'s search query.
+If you have more complex criteria, you can always apply the filter to the resulting Pandas `DataFrame`.
+
+```python
+
+# Apply the filter via LanceDB
+results = table.search([100, 100]).where("price < 15").to_df()
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+
+# Apply the filter via Pandas
+df = results = table.search([100, 100]).to_df()
+results = df[df.price < 15]
+assert len(results) == 1
+assert results["item"].iloc[0] == "foo"
+```

docs/src/python/duckdb.md

@@ -0,0 +1,56 @@
+# DuckDB
+
+`LanceDB` works with `DuckDB` via [PyArrow integration](https://duckdb.org/docs/guides/python/sql_on_arrow).
+
+Let us start with installing `duckdb` and `lancedb`.
+
+```shell
+pip install duckdb lancedb
+```
+
+We will re-use [the dataset created previously](./arrow.md):
+
+```python
+import pandas as pd
+import lancedb
+
+db = lancedb.connect("data/sample-lancedb")
+data = pd.DataFrame({
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "item": ["foo", "bar"],
+    "price": [10.0, 20.0]
+})
+table = db.create_table("pd_table", data=data)
+arrow_table = table.to_arrow()
+```
+
+`DuckDB` can directly query the `arrow_table`:
+
+```python
+import duckdb
+
+duckdb.query("SELECT * FROM arrow_table")
+```
+
+```
+┌─────────────┬─────────┬────────┐
+│   vector    │  item   │ price  │
+│   float[]   │ varchar │ double │
+├─────────────┼─────────┼────────┤
+│ [3.1, 4.1]  │ foo     │   10.0 │
+│ [5.9, 26.5] │ bar     │   20.0 │
+└─────────────┴─────────┴────────┘
+```
+
+```py
+duckdb.query("SELECT mean(price) FROM arrow_table")
+```
+
+```
+┌─────────────┐
+│ mean(price) │
+│   double    │
+├─────────────┤
+│        15.0 │
+└─────────────┘
+```

docs/src/python/integration.md

@@ -0,0 +1,7 @@
+# Integration
+
+Built on top of [Apache Arrow](https://arrow.apache.org/),
+`LanceDB` is very easy to be integrate with Python ecosystems.
+
+* [Pandas and Arrow Integration](./arrow.md)
+* [DuckDB Integration](./duckdb.md)

docs/src/python/pydantic.md

@@ -0,0 +1,35 @@
+# Pydantic
+
+[Pydantic](https://docs.pydantic.dev/latest/) is a data validation library in Python.
+
+## Schema
+
+LanceDB supports to create Apache Arrow Schema from a
+[Pydantic BaseModel](https://docs.pydantic.dev/latest/api/main/#pydantic.main.BaseModel)
+via [pydantic_to_schema()](python.md##lancedb.pydantic.pydantic_to_schema) method.
+
+::: lancedb.pydantic.pydantic_to_schema
+
+## Vector Field
+
+LanceDB provides a [`vector(dim)`](python.md#lancedb.pydantic.vector) method to define a
+vector Field in a Pydantic Model.
+
+::: lancedb.pydantic.vector
+
+## Type Conversion
+
+LanceDB automatically convert Pydantic fields to
+[Apache Arrow DataType](https://arrow.apache.org/docs/python/generated/pyarrow.DataType.html#pyarrow.DataType).
+
+Current supported type conversions:
+
+| Pydantic Field Type | PyArrow Data Type |
+| ------------------- | ----------------- |
+| `int`               | `pyarrow.int64`   |
+| `float`              | `pyarrow.float64`  |
+| `bool`              | `pyarrow.bool`    |
+| `str`               | `pyarrow.utf8()`    |
+| `list`              | `pyarrow.List`    |
+| `BaseModel`         | `pyarrow.Struct`    |
+| `vector(n)`         | `pyarrow.FixedSizeList(float32, n)` |

docs/src/python/python.md

@@ -43,3 +43,17 @@ pip install lancedb
 ::: lancedb.fts.populate_index
 
 ::: lancedb.fts.search_index
+
+## Utilities
+
+::: lancedb.vector
+
+## Integrations
+
+### Pydantic
+
+::: lancedb.pydantic.pydantic_to_schema
+
+::: lancedb.pydantic.vector
+
+

docs/src/search.md

@@ -25,9 +25,9 @@ Currently, we support the following metrics:
 
 ### Flat Search
 
+If LanceDB does not create a vector index, LanceDB would need to scan (`Flat Search`) the entire vector column
+and compute the distance for each vector in order to find the closest matches.
 
-If there is no [vector index is created](ann_indexes.md), LanceDB will just brute-force scan
-the vector column and compute the distance.
 
 <!-- Setup Code
 ```python
@@ -79,39 +79,43 @@ await db_setup.createTable('my_vectors', data)
     const tbl = await db.openTable("my_vectors")
 
     const results_1 = await tbl.search(Array(1536).fill(1.2))
-        .limit(20)
+        .limit(10)
         .execute()
     ```
 
-
-<!-- Commenting out for now since metricType fails for JS on Ubuntu 22.04.
-
 By default, `l2` will be used as `Metric` type. You can customize the metric type
 as well.
--->
 
-<!--
 === "Python"
--->
-<!--    ```python
+
+    ```python
     df = tbl.search(np.random.random((1536))) \
         .metric("cosine") \
         .limit(10) \
         .to_df()
     ```
--->
-<!--
-=== "JavaScript"
--->
 
-<!--   ```javascript
+
+=== "JavaScript"
+
+    ```javascript
     const results_2 = await tbl.search(Array(1536).fill(1.2))
         .metricType("cosine")
-        .limit(20)
+        .limit(10)
         .execute()
     ```
--->
 
-### Search with Vector Index.
+
+### Approximate Nearest Neighbor (ANN) Search with Vector Index.
+
+To accelerate vector retrievals, it is common to build vector indices.
+A vector index is a data structure specifically designed to efficiently organize and
+search vector data based on their similarity or distance metrics.
+By constructing a vector index, you can reduce the search space and avoid the need
+for brute-force scanning of the entire vector column.
+
+However, fast vector search using indices often entails making a trade-off with accuracy to some extent.
+This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
+always returns 100% recall.
 
 See [ANN Index](ann_indexes.md) for more details.

docs/test/md_testing.js

@@ -7,6 +7,7 @@ const excludedFiles = [
   "../src/embedding.md",
   "../src/examples/serverless_lancedb_with_s3_and_lambda.md",
   "../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
+  "../src/examples/transformerjs_embedding_search_nodejs.md",
   "../src/examples/youtube_transcript_bot_with_nodejs.md",
 ];
 const nodePrefix = "javascript";
@@ -48,4 +49,4 @@ for (const file of files.filter((file) => !excludedFiles.includes(file))) {
     fs.mkdirSync(path.dirname(outPath), { recursive: true });
     fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
   }
-}
+}

node/.npmignore

@@ -0,0 +1,4 @@
+gen_test_data.py
+index.node
+dist/lancedb*.tgz
+vectordb*.tgz

node/README.md

@@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
@@ -26,12 +30,34 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-Run the tests with
+To build everything fresh:
+
+```bash
+npm install
+npm run tsc
+npm run build
+```
+
+Then you should be able to run the tests with:
 
 ```bash
 npm test
 ```
 
+### Rebuilding Rust library
+
+```bash
+npm run build
+```
+
+### Rebuilding Typescript
+
+```bash
+npm run tsc
+```
+
+### Fix lints
+
 To run the linter and have it automatically fix all errors
 
 ```bash

node/examples/js-transformers/index.js

@@ -0,0 +1,66 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict'
+
+
+async function example() {
+
+    const lancedb = require('vectordb')
+
+    // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
+    const { pipeline } = await import('@xenova/transformers')
+    const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+
+
+    // Create embedding function from pipeline which returns a list of vectors from batch
+    // sourceColumn is the name of the column in the data to be embedded
+    //
+    // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
+    const embed_fun = {}
+    embed_fun.sourceColumn = 'text'
+    embed_fun.embed = async function (batch) {
+        let result = []
+        for (let text of batch) {
+            const res = await pipe(text, { pooling: 'mean', normalize: true })
+            result.push(Array.from(res['data']))
+        }
+        return (result)
+    }
+
+    // Link a folder and create a table with data
+    const db = await lancedb.connect('data/sample-lancedb')
+
+    const data = [
+        { id: 1, text: 'Cherry', type: 'fruit' },
+        { id: 2, text: 'Carrot', type: 'vegetable' },
+        { id: 3, text: 'Potato', type: 'vegetable' },
+        { id: 4, text: 'Apple', type: 'fruit' },
+        { id: 5, text: 'Banana', type: 'fruit' }
+    ]
+
+    const table = await db.createTable('food_table', data, "create", embed_fun)
+
+
+    // Query the table
+    const results = await table
+        .search("a sweet fruit to eat")
+        .metricType("cosine")
+        .limit(2)
+        .execute()
+    console.log(results.map(r => r.text))
+
+}
+
+example().then(_ => { console.log("Done!") })

node/examples/js-transformers/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "vectordb-example-js-transformers",
+  "version": "1.0.0",
+  "description": "Example for using transformers.js with lancedb",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@xenova/transformers": "^2.4.1",
+    "vectordb": "^0.1.12"
+  }
+
+}

node/native.js

@@ -12,29 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;
 
-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`vectordb-${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    try {
+        // Might be developing locally, so try that. But don't expose that error
+        // to the user.
+        nativeLib = require("./index.node");
+    } catch {
+        throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install vectordb-${currentTarget()}\`.
+      
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+      
+  Source error: ${e}`);
+    }   
 }
 
-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;

node/package-lock.json

@@ -1,18 +1,30 @@
 {
   "name": "vectordb",
-  "version": "0.1.9",
+  "version": "0.1.14",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.9",
+      "version": "0.1.14",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
       "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
-        "apache-arrow": "^12.0.0"
+        "@neon-rs/load": "^0.0.74",
+        "apache-arrow": "^12.0.0",
+        "axios": "^1.4.0"
       },
       "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/chai-as-promised": "^7.1.5",
         "@types/mocha": "^10.0.1",
@@ -37,6 +49,13 @@
         "typedoc": "^0.24.7",
         "typedoc-plugin-markdown": "^3.15.3",
         "typescript": "*"
+      },
+      "optionalDependencies": {
+        "vectordb-darwin-arm64": "0.1.14",
+        "vectordb-darwin-x64": "0.1.14",
+        "vectordb-linux-arm64-gnu": "0.1.14",
+        "vectordb-linux-x64-gnu": "0.1.14",
+        "vectordb-win32-x64-msvc": "0.1.14"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -204,6 +223,20 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -810,8 +843,7 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
@@ -826,12 +858,13 @@
       }
     },
     "node_modules/axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
       "dependencies": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "node_modules/balanced-match": {
@@ -1062,7 +1095,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -1285,7 +1317,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2052,7 +2083,6 @@
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
       "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true,
       "funding": [
         {
           "type": "individual",
@@ -2081,7 +2111,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -2955,7 +2984,6 @@
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
       "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
       "engines": {
         "node": ">= 0.6"
       }
@@ -2964,7 +2992,6 @@
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "dependencies": {
         "mime-db": "1.52.0"
       },
@@ -3258,6 +3285,15 @@
         "form-data": "^4.0.0"
       }
     },
+    "node_modules/openai/node_modules/axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
     "node_modules/optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -3409,6 +3445,11 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "node_modules/punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -4256,6 +4297,42 @@
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
+    "node_modules/vectordb-darwin-arm64": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.14.tgz",
+      "integrity": "sha512-5doSFMUR4scxseo73thCxScmO3Wpb+cqPsIa7+2uneTEtBSViMbkw/1mGTC+rV4NTCnxhoiqHk9pJzZVeDMkPg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/vectordb-darwin-x64": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.14.tgz",
+      "integrity": "sha512-x+qVaKNhAG65HdENL6GRJjxl1hZ7erRm3a2rhplyYoQyzuRPPBILeWzxkE01G1fb0+47dehe7Q4f/8BDaghcCQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/vectordb-linux-x64-gnu": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.14.tgz",
+      "integrity": "sha512-hvA2YYwTZK92k6nPH99Jn5N0CwagDOdnwMmjtCpzFOEYK7dY/2kcTOoQNlBwwNP9MYvgN6jdFD/Cwkih1X/qjA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
     "node_modules/vscode-oniguruma": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",
@@ -4601,6 +4678,17 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -5056,8 +5144,7 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
     },
     "available-typed-arrays": {
       "version": "1.0.5",
@@ -5066,12 +5153,13 @@
       "dev": true
     },
     "axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz",
+      "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==",
       "requires": {
-        "follow-redirects": "^1.14.8"
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
       }
     },
     "balanced-match": {
@@ -5251,7 +5339,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
       }
@@ -5418,8 +5505,7 @@
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
     },
     "diff": {
       "version": "4.0.2",
@@ -5989,8 +6075,7 @@
     "follow-redirects": {
       "version": "1.15.2",
       "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
     },
     "for-each": {
       "version": "0.3.3",
@@ -6005,7 +6090,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -6619,14 +6703,12 @@
     "mime-db": {
       "version": "1.52.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
     },
     "mime-types": {
       "version": "2.1.35",
       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
       "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
       "requires": {
         "mime-db": "1.52.0"
       }
@@ -6852,6 +6934,17 @@
       "requires": {
         "axios": "^0.26.0",
         "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.26.1",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+          "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+          "dev": true,
+          "requires": {
+            "follow-redirects": "^1.14.8"
+          }
+        }
       }
     },
     "optionator": {
@@ -6960,6 +7053,11 @@
       "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==",
       "dev": true
     },
+    "proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "punycode": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz",
@@ -7540,6 +7638,24 @@
       "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
       "dev": true
     },
+    "vectordb-darwin-arm64": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.1.14.tgz",
+      "integrity": "sha512-5doSFMUR4scxseo73thCxScmO3Wpb+cqPsIa7+2uneTEtBSViMbkw/1mGTC+rV4NTCnxhoiqHk9pJzZVeDMkPg==",
+      "optional": true
+    },
+    "vectordb-darwin-x64": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-darwin-x64/-/vectordb-darwin-x64-0.1.14.tgz",
+      "integrity": "sha512-x+qVaKNhAG65HdENL6GRJjxl1hZ7erRm3a2rhplyYoQyzuRPPBILeWzxkE01G1fb0+47dehe7Q4f/8BDaghcCQ==",
+      "optional": true
+    },
+    "vectordb-linux-x64-gnu": {
+      "version": "0.1.14",
+      "resolved": "https://registry.npmjs.org/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.1.14.tgz",
+      "integrity": "sha512-hvA2YYwTZK92k6nPH99Jn5N0CwagDOdnwMmjtCpzFOEYK7dY/2kcTOoQNlBwwNP9MYvgN6jdFD/Cwkih1X/qjA==",
+      "optional": true
+    },
     "vscode-oniguruma": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz",

node/package.json

@@ -1,16 +1,18 @@
 {
   "name": "vectordb",
-  "version": "0.1.10",
+  "version": "0.1.14",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
-    "test": "npm run tsc; mocha -recursive dist/test",
+    "test": "npm run tsc && mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "clean": "rm -rf node_modules *.node dist/"
+    "clean": "rm -rf node_modules *.node dist/",
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
   },
   "repository": {
     "type": "git",
@@ -25,6 +27,7 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.1",
@@ -52,6 +55,33 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
-    "apache-arrow": "^12.0.0"
+    "@neon-rs/load": "^0.0.74",
+    "apache-arrow": "^12.0.0",
+    "axios": "^1.4.0"
+  },
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "vectordb-darwin-x64",
+      "aarch64-apple-darwin": "vectordb-darwin-arm64",
+      "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu",
+      "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu",
+      "x86_64-pc-windows-msvc": "vectordb-win32-x64-msvc"
+    }
+  },
+  "optionalDependencies": {
+    "vectordb-darwin-arm64": "0.1.14",
+    "vectordb-darwin-x64": "0.1.14",
+    "vectordb-linux-arm64-gnu": "0.1.14",
+    "vectordb-linux-x64-gnu": "0.1.14",
+    "vectordb-win32-x64-msvc": "0.1.14"
   }
 }

node/src/index.ts

@@ -14,26 +14,64 @@
 
 import {
   RecordBatchFileWriter,
-  type Table as ArrowTable,
-  tableFromIPC,
-  Vector
+  type Table as ArrowTable
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
 import type { EmbeddingFunction } from './embedding/embedding_function'
+import { RemoteConnection } from './remote'
+import { Query } from './query'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete } = require('../native.js')
 
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
+export interface AwsCredentials {
+  accessKeyId: string
+
+  secretKey: string
+
+  sessionToken?: string
+}
+
+export interface ConnectionOptions {
+  uri: string
+
+  awsCredentials?: AwsCredentials
+
+  // API key for the remote connections
+  apiKey?: string
+  // Region to connect
+  region?: string
+}
+
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
-export async function connect (uri: string): Promise<Connection> {
-  const db = await databaseNew(uri)
-  return new LocalConnection(db, uri)
+export async function connect (uri: string): Promise<Connection>
+export async function connect (opts: Partial<ConnectionOptions>): Promise<Connection>
+export async function connect (arg: string | Partial<ConnectionOptions>): Promise<Connection> {
+  let opts: ConnectionOptions
+  if (typeof arg === 'string') {
+    opts = { uri: arg }
+  } else {
+    // opts = { uri: arg.uri, awsCredentials = arg.awsCredentials }
+    opts = Object.assign({
+      uri: '',
+      awsCredentials: undefined,
+      apiKey: undefined,
+      region: 'us-west-2'
+    }, arg)
+  }
+
+  if (opts.uri.startsWith('db://')) {
+    // Remote connection
+    return new RemoteConnection(opts)
+  }
+  const db = await databaseNew(opts.uri)
+  return new LocalConnection(db, opts)
 }
 
 /**
@@ -117,7 +155,34 @@ export interface Table<T = number[]> {
   /**
    * Delete rows from this table.
    *
-   * @param filter  A filter in the same format used by a sql WHERE clause.
+   * This can be used to delete a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause. The
+   *                filter must not be empty.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [1, 2]},
+   *    {id: 2, vector: [3, 4]},
+   *    {id: 3, vector: [5, 6]},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   * await tbl.delete("id = 2")
+   * await tbl.countRows() // Returns 2
+   * ```
+   *
+   * If you have a list of values to delete, you can combine them into a
+   * stringified list and use the `IN` operator:
+   *
+   * ```ts
+   * const to_remove = [1, 5];
+   * await tbl.delete(`id IN (${to_remove.join(",")})`)
+   * await tbl.countRows() // Returns 1
+   * ```
    */
   delete: (filter: string) => Promise<void>
 }
@@ -126,21 +191,21 @@ export interface Table<T = number[]> {
  * A connection to a LanceDB database.
  */
 export class LocalConnection implements Connection {
-  private readonly _uri: string
+  private readonly _options: ConnectionOptions
   private readonly _db: any
 
-  constructor (db: any, uri: string) {
-    this._uri = uri
+  constructor (db: any, options: ConnectionOptions) {
+    this._options = options
     this._db = db
   }
 
   get uri (): string {
-    return this._uri
+    return this._options.uri
   }
 
   /**
-     * Get the names of all tables in the database.
-     */
+   * Get the names of all tables in the database.
+   */
   async tableNames (): Promise<string[]> {
     return databaseTableNames.call(this._db)
   }
@@ -151,6 +216,7 @@ export class LocalConnection implements Connection {
    * @param name The name of the table.
    */
   async openTable (name: string): Promise<Table>
+
   /**
    * Open a table in the database.
    *
@@ -158,12 +224,13 @@ export class LocalConnection implements Connection {
    * @param embeddings An embedding function to use on this Table
    */
   async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
     const tbl = await databaseOpenTable.call(this._db, name)
     if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, embeddings)
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      return new LocalTable(tbl, name)
+      return new LocalTable(tbl, name, this._options)
     }
   }
 
@@ -186,15 +253,27 @@ export class LocalConnection implements Connection {
    * @param embeddings An embedding function to use on this Table
    */
   async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
   async createTable<T> (name: string, data: Array<Record<string, unknown>>, mode: WriteMode, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
     if (mode === undefined) {
       mode = WriteMode.Create
     }
-    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase())
+
+    const createArgs = [this._db, name, await fromRecordsToBuffer(data, embeddings), mode.toLowerCase()]
+    if (this._options.awsCredentials !== undefined) {
+      createArgs.push(this._options.awsCredentials.accessKeyId)
+      createArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        createArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+
+    const tbl = await tableCreate.call(...createArgs)
+
     if (embeddings !== undefined) {
-      return new LocalTable(tbl, name, embeddings)
+      return new LocalTable(tbl, name, this._options, embeddings)
     } else {
-      return new LocalTable(tbl, name)
+      return new LocalTable(tbl, name, this._options)
     }
   }
 
@@ -217,18 +296,21 @@ export class LocalTable<T = number[]> implements Table<T> {
   private readonly _tbl: any
   private readonly _name: string
   private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _options: ConnectionOptions
 
-  constructor (tbl: any, name: string)
+  constructor (tbl: any, name: string, options: ConnectionOptions)
   /**
    * @param tbl
    * @param name
+   * @param options
    * @param embeddings An embedding function to use when interacting with this table
    */
-  constructor (tbl: any, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings: EmbeddingFunction<T>)
+  constructor (tbl: any, name: string, options: ConnectionOptions, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._name = name
     this._embeddings = embeddings
+    this._options = options
   }
 
   get name (): string {
@@ -240,7 +322,7 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @param query The query search term
    */
   search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+    return new Query(query, this._tbl, this._embeddings)
   }
 
   /**
@@ -250,7 +332,15 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString())
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
+    return tableAdd.call(...callArgs)
   }
 
   /**
@@ -260,6 +350,14 @@ export class LocalTable<T = number[]> implements Table<T> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
+    const callArgs = [this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString()]
+    if (this._options.awsCredentials !== undefined) {
+      callArgs.push(this._options.awsCredentials.accessKeyId)
+      callArgs.push(this._options.awsCredentials.secretKey)
+      if (this._options.awsCredentials.sessionToken !== undefined) {
+        callArgs.push(this._options.awsCredentials.sessionToken)
+      }
+    }
     return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
   }
 
@@ -346,116 +444,6 @@ export interface IvfPQIndexConfig {
 
 export type VectorIndexParams = IvfPQIndexConfig
 
-/**
- * A builder for nearest neighbor queries for LanceDB.
- */
-export class Query<T = number[]> {
-  private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
-  private _limit: number
-  private _refineFactor?: number
-  private _nprobes: number
-  private _select?: string[]
-  private _filter?: string
-  private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
-
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
-    this._tbl = tbl
-    this._query = query
-    this._limit = 10
-    this._nprobes = 20
-    this._refineFactor = undefined
-    this._select = undefined
-    this._filter = undefined
-    this._metricType = undefined
-    this._embeddings = embeddings
-  }
-
-  /***
-   * Sets the number of results that will be returned
-   * @param value number of results
-   */
-  limit (value: number): Query<T> {
-    this._limit = value
-    return this
-  }
-
-  /**
-   * Refine the results by reading extra elements and re-ranking them in memory.
-   * @param value refine factor to use in this query.
-   */
-  refineFactor (value: number): Query<T> {
-    this._refineFactor = value
-    return this
-  }
-
-  /**
-   * The number of probes used. A higher number makes search more accurate but also slower.
-   * @param value The number of probes used.
-   */
-  nprobes (value: number): Query<T> {
-    this._nprobes = value
-    return this
-  }
-
-  /**
-   * A filter statement to be applied to this query.
-   * @param value A filter in the same format used by a sql WHERE clause.
-   */
-  filter (value: string): Query<T> {
-    this._filter = value
-    return this
-  }
-
-  where = this.filter
-
-  /** Return only the specified columns.
-   *
-   * @param value Only select the specified columns. If not specified, all columns will be returned.
-   */
-  select (value: string[]): Query<T> {
-    this._select = value
-    return this
-  }
-
-  /**
-   * The MetricType used for this Query.
-   * @param value The metric to the. @see MetricType for the different options
-   */
-  metricType (value: MetricType): Query<T> {
-    this._metricType = value
-    return this
-  }
-
-  /**
-   * Execute the query and return the results as an Array of Objects
-   */
-  async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
-    const buffer = await tableSearch.call(this._tbl, this)
-    const data = tableFromIPC(buffer)
-
-    return data.toArray().map((entry: Record<string, unknown>) => {
-      const newObject: Record<string, unknown> = {}
-      Object.keys(entry).forEach((key: string) => {
-        if (entry[key] instanceof Vector) {
-          newObject[key] = (entry[key] as Vector).toArray()
-        } else {
-          newObject[key] = entry[key]
-        }
-      })
-      return newObject as unknown as T
-    })
-  }
-}
-
 /**
  * Write mode for writing a table.
  */

node/src/query.ts

@@ -0,0 +1,130 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Vector, tableFromIPC } from 'apache-arrow'
+import { type EmbeddingFunction } from './embedding/embedding_function'
+import { type MetricType } from '.'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { tableSearch } = require('../native.js')
+
+/**
+ * A builder for nearest neighbor queries for LanceDB.
+ */
+export class Query<T = number[]> {
+  private readonly _query: T
+  private readonly _tbl?: any
+  private _queryVector?: number[]
+  private _limit: number
+  private _refineFactor?: number
+  private _nprobes: number
+  private _select?: string[]
+  private _filter?: string
+  private _metricType?: MetricType
+  protected readonly _embeddings?: EmbeddingFunction<T>
+
+  constructor (query: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
+    this._tbl = tbl
+    this._query = query
+    this._limit = 10
+    this._nprobes = 20
+    this._refineFactor = undefined
+    this._select = undefined
+    this._filter = undefined
+    this._metricType = undefined
+    this._embeddings = embeddings
+  }
+
+  /***
+     * Sets the number of results that will be returned
+     * @param value number of results
+     */
+  limit (value: number): Query<T> {
+    this._limit = value
+    return this
+  }
+
+  /**
+     * Refine the results by reading extra elements and re-ranking them in memory.
+     * @param value refine factor to use in this query.
+     */
+  refineFactor (value: number): Query<T> {
+    this._refineFactor = value
+    return this
+  }
+
+  /**
+     * The number of probes used. A higher number makes search more accurate but also slower.
+     * @param value The number of probes used.
+     */
+  nprobes (value: number): Query<T> {
+    this._nprobes = value
+    return this
+  }
+
+  /**
+     * A filter statement to be applied to this query.
+     * @param value A filter in the same format used by a sql WHERE clause.
+     */
+  filter (value: string): Query<T> {
+    this._filter = value
+    return this
+  }
+
+  where = this.filter
+
+  /** Return only the specified columns.
+     *
+     * @param value Only select the specified columns. If not specified, all columns will be returned.
+     */
+  select (value: string[]): Query<T> {
+    this._select = value
+    return this
+  }
+
+  /**
+     * The MetricType used for this Query.
+     * @param value The metric to the. @see MetricType for the different options
+     */
+  metricType (value: MetricType): Query<T> {
+    this._metricType = value
+    return this
+  }
+
+  /**
+     * Execute the query and return the results as an Array of Objects
+     */
+  async execute<T = Record<string, unknown>> (): Promise<T[]> {
+    if (this._embeddings !== undefined) {
+      this._queryVector = (await this._embeddings.embed([this._query]))[0]
+    } else {
+      this._queryVector = this._query as number[]
+    }
+
+    const buffer = await tableSearch.call(this._tbl, this)
+    const data = tableFromIPC(buffer)
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}

node/src/remote/client.ts

@@ -0,0 +1,100 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios, { type AxiosResponse } from 'axios'
+
+import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'
+
+export class HttpLancedbClient {
+  private readonly _url: string
+
+  public constructor (url: string, private readonly _apiKey: string) {
+    this._url = url
+  }
+
+  get uri (): string {
+    return this._url
+  }
+
+  public async search (
+    tableName: string,
+    vector: number[],
+    k: number,
+    nprobes: number,
+    refineFactor?: number,
+    columns?: string[],
+    filter?: string
+  ): Promise<ArrowTable<any>> {
+    const response = await axios.post(
+              `${this._url}/v1/table/${tableName}`,
+              {
+                vector,
+                k,
+                nprobes,
+                refineFactor,
+                columns,
+                filter
+              },
+              {
+                headers: {
+                  'Content-Type': 'application/json',
+                  'x-api-key': this._apiKey
+                },
+                responseType: 'arraybuffer',
+                timeout: 10000
+              }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+
+    const table = tableFromIPC(response.data)
+    return table
+  }
+
+  /**
+   * Sent GET request.
+   */
+  public async get (path: string, params?: Record<string, string | number>): Promise<AxiosResponse> {
+    const response = await axios.get(
+      `${this._url}${path}`,
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this._apiKey
+        },
+        params,
+        timeout: 10000
+      }
+    ).catch((err) => {
+      console.error('error: ', err)
+      return err.response
+    })
+    if (response.status !== 200) {
+      const errorData = new TextDecoder().decode(response.data)
+      throw new Error(
+        `Server Error, status: ${response.status as number}, ` +
+        `message: ${response.statusText as string}: ${errorData}`
+      )
+    }
+    return response
+  }
+}

node/src/remote/index.ts

@@ -0,0 +1,163 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
+  type ConnectionOptions
+} from '../index'
+import { Query } from '../query'
+
+import { type Table as ArrowTable, Vector } from 'apache-arrow'
+import { HttpLancedbClient } from './client'
+
+/**
+ * Remote connection.
+ */
+export class RemoteConnection implements Connection {
+  private readonly _client: HttpLancedbClient
+  private readonly _dbName: string
+
+  constructor (opts: ConnectionOptions) {
+    if (!opts.uri.startsWith('db://')) {
+      throw new Error(`Invalid remote DB URI: ${opts.uri}`)
+    }
+    if (opts.apiKey === undefined || opts.region === undefined) {
+      throw new Error('API key and region are not supported for remote connections')
+    }
+
+    this._dbName = opts.uri.slice('db://'.length)
+    const server = `https://${this._dbName}.${opts.region}.api.lancedb.com`
+    this._client = new HttpLancedbClient(server, opts.apiKey)
+  }
+
+  get uri (): string {
+    // add the lancedb+ prefix back
+    return 'db://' + this._client.uri
+  }
+
+  async tableNames (): Promise<string[]> {
+    const response = await this._client.get('/v1/table/')
+    return response.data.tables
+  }
+
+  async openTable (name: string): Promise<Table>
+  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    if (embeddings !== undefined) {
+      return new RemoteTable(this._client, name, embeddings)
+    } else {
+      return new RemoteTable(this._client, name)
+    }
+  }
+
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+    throw new Error('Not implemented')
+  }
+
+  async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
+    throw new Error('Not implemented')
+  }
+
+  async dropTable (name: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+}
+
+export class RemoteQuery<T = number[]> extends Query<T> {
+  constructor (query: T, private readonly _client: HttpLancedbClient,
+    private readonly _name: string, embeddings?: EmbeddingFunction<T>) {
+    super(query, undefined, embeddings)
+  }
+
+  // TODO: refactor this to a base class + queryImpl pattern
+  async execute<T = Record<string, unknown>>(): Promise<T[]> {
+    const embeddings = this._embeddings
+    const query = (this as any)._query
+    let queryVector: number[]
+
+    if (embeddings !== undefined) {
+      queryVector = (await embeddings.embed([query]))[0]
+    } else {
+      queryVector = query as number[]
+    }
+
+    const data = await this._client.search(
+      this._name,
+      queryVector,
+      (this as any)._limit,
+      (this as any)._nprobes,
+      (this as any)._refineFactor,
+      (this as any)._select,
+      (this as any)._filter
+    )
+
+    return data.toArray().map((entry: Record<string, unknown>) => {
+      const newObject: Record<string, unknown> = {}
+      Object.keys(entry).forEach((key: string) => {
+        if (entry[key] instanceof Vector) {
+          newObject[key] = (entry[key] as Vector).toArray()
+        } else {
+          newObject[key] = entry[key]
+        }
+      })
+      return newObject as unknown as T
+    })
+  }
+}
+
+// we are using extend until we have next next version release
+// Table and Connection has both been refactored to interfaces
+export class RemoteTable<T = number[]> implements Table<T> {
+  private readonly _client: HttpLancedbClient
+  private readonly _embeddings?: EmbeddingFunction<T>
+  private readonly _name: string
+
+  constructor (client: HttpLancedbClient, name: string)
+  constructor (client: HttpLancedbClient, name: string, embeddings: EmbeddingFunction<T>)
+  constructor (client: HttpLancedbClient, name: string, embeddings?: EmbeddingFunction<T>) {
+    this._client = client
+    this._name = name
+    this._embeddings = embeddings
+  }
+
+  get name (): string {
+    return this._name
+  }
+
+  search (query: T): Query<T> {
+    return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
+  }
+
+  async add (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+    throw new Error('Not implemented')
+  }
+
+  async countRows (): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async delete (filter: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+}

node/src/test/io.ts

@@ -18,26 +18,48 @@ import { describe } from 'mocha'
 import { assert } from 'chai'
 
 import * as lancedb from '../index'
+import { type ConnectionOptions } from '../index'
 
 describe('LanceDB S3 client', function () {
   if (process.env.TEST_S3_BASE_URL != null) {
     const baseUri = process.env.TEST_S3_BASE_URL
     it('should have a valid url', async function () {
-      const uri = `${baseUri}/valid_url`
-      const table = await createTestDB(uri, 2, 20)
-      const con = await lancedb.connect(uri)
-      assert.equal(con.uri, uri)
+      const opts = { uri: `${baseUri}/valid_url` }
+      const table = await createTestDB(opts, 2, 20)
+      const con = await lancedb.connect(opts)
+      assert.equal(con.uri, opts.uri)
 
       const results = await table.search([0.1, 0.3]).limit(5).execute()
       assert.equal(results.length, 5)
-    })
+    }).timeout(10_000)
+  } else {
+    describe.skip('Skip S3 test', function () {})
+  }
+
+  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
+    const baseUri = process.env.TEST_S3_BASE_URL
+    it('use custom credentials', async function () {
+      const opts: ConnectionOptions = {
+        uri: `${baseUri}/custom_credentials`,
+        awsCredentials: {
+          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
+          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
+        }
+      }
+      const table = await createTestDB(opts, 2, 20)
+      const con = await lancedb.connect(opts)
+      assert.equal(con.uri, opts.uri)
+
+      const results = await table.search([0.1, 0.3]).limit(5).execute()
+      assert.equal(results.length, 5)
+    }).timeout(10_000)
   } else {
     describe.skip('Skip S3 test', function () {})
   }
 })
 
-async function createTestDB (uri: string, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(uri)
+async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
+  const con = await lancedb.connect(opts)
 
   const data = []
   for (let i = 0; i < numRows; i++) {

node/src/test/test.ts

@@ -18,7 +18,8 @@ import * as chai from 'chai'
 import * as chaiAsPromised from 'chai-as-promised'
 
 import * as lancedb from '../index'
-import { type EmbeddingFunction, MetricType, Query, WriteMode } from '../index'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, WriteMode } from '../index'
+import { Query } from '../query'
 
 const expect = chai.expect
 const assert = chai.assert
@@ -32,6 +33,22 @@ describe('LanceDB client', function () {
       assert.equal(con.uri, uri)
     })
 
+    it('should accept an options object', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect({ uri })
+      assert.equal(con.uri, uri)
+    })
+
+    it('should accept custom aws credentials', async function () {
+      const uri = await createTestDB()
+      const awsCredentials: AwsCredentials = {
+        accessKeyId: '',
+        secretKey: ''
+      }
+      const con = await lancedb.connect({ uri, awsCredentials })
+      assert.equal(con.uri, uri)
+    })
+
     it('should return the existing table names', async function () {
       const uri = await createTestDB()
       const con = await lancedb.connect(uri)
@@ -252,7 +269,7 @@ describe('LanceDB client', function () {
 
 describe('Query object', function () {
   it('sets custom parameters', async function () {
-    const query = new Query(undefined, [0.1, 0.3])
+    const query = new Query([0.1, 0.3])
       .limit(1)
       .metricType(MetricType.Cosine)
       .refineFactor(100)

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.8
+current_version = 0.1.12
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/__init__.py

@@ -15,6 +15,7 @@ from typing import Optional
 
 from .db import URI, DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
+from .schema import vector
 
 
 def connect(

python/lancedb/db.py

@@ -13,17 +13,18 @@
 
 from __future__ import annotations
 
-import functools
 import os
 from abc import ABC, abstractmethod
 from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 
+import pandas as pd
 import pyarrow as pa
 from pyarrow import fs
 
 from .common import DATA, URI
 from .table import LanceTable, Table
-from .util import get_uri_location, get_uri_scheme
+from .util import fs_from_uri, get_uri_location, get_uri_scheme
 
 
 class DBConnection(ABC):
@@ -38,8 +39,10 @@ class DBConnection(ABC):
     def create_table(
         self,
         name: str,
-        data: DATA = None,
-        schema: pa.Schema = None,
+        data: Optional[
+            Union[List[dict], dict, pd.DataFrame, pa.Table, Iterable[pa.RecordBatch]],
+        ] = None,
+        schema: Optional[pa.Schema] = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
@@ -51,7 +54,7 @@ class DBConnection(ABC):
         name: str
             The name of the table.
         data: list, tuple, dict, pd.DataFrame; optional
-            The data to insert into the table.
+            The data to initialize the table. User must provide at least one of `data` or `schema`.
         schema: pyarrow.Schema; optional
             The schema of the table.
         mode: str; default "create"
@@ -64,16 +67,16 @@ class DBConnection(ABC):
         fill_value: float
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
-        Note
-        ----
-        The vector index won't be created by default.
-        To create the index, call the `create_index` method on the table.
-
         Returns
         -------
         LanceTable
             A reference to the newly created table.
 
+        !!! note
+
+            The vector index won't be created by default.
+            To create the index, call the `create_index` method on the table.
+
         Examples
         --------
 
@@ -119,7 +122,7 @@ class DBConnection(ABC):
 
         Data is converted to Arrow before being written to disk. For maximum
         control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a PyArrow table directly.
+        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.
 
         >>> custom_schema = pa.schema([
         ...   pa.field("vector", pa.list_(pa.float32(), 2)),
@@ -138,6 +141,30 @@ class DBConnection(ABC):
         vector: [[[1.1,1.2],[0.2,1.8]]]
         lat: [[45.5,40.1]]
         long: [[-122.7,-74.1]]
+
+
+        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:
+
+
+        >>> import pyarrow as pa
+        >>> def make_batches():
+        ...     for i in range(5):
+        ...         yield pa.RecordBatch.from_arrays(
+        ...             [
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]]),
+        ...                 pa.array(["foo", "bar"]),
+        ...                 pa.array([10.0, 20.0]),
+        ...             ],
+        ...             ["vector", "item", "price"],
+        ...         )
+        >>> schema=pa.schema([
+        ...     pa.field("vector", pa.list_(pa.float32())),
+        ...     pa.field("item", pa.utf8()),
+        ...     pa.field("price", pa.float32()),
+        ... ])
+        >>> db.create_table("table4", make_batches(), schema=schema)
+        LanceTable(table4)
+
         """
         raise NotImplementedError
 
@@ -225,7 +252,7 @@ class LanceDBConnection(DBConnection):
             A list of table names.
         """
         try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
+            filesystem, path = fs_from_uri(self.uri)
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
@@ -252,7 +279,7 @@ class LanceDBConnection(DBConnection):
     def create_table(
         self,
         name: str,
-        data: DATA = None,
+        data: Optional[Union[List[dict], dict, pd.DataFrame]] = None,
         schema: pa.Schema = None,
         mode: str = "create",
         on_bad_vectors: str = "error",
@@ -260,114 +287,22 @@ class LanceDBConnection(DBConnection):
     ) -> LanceTable:
         """Create a table in the database.
 
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to insert into the table.
-        schema: pyarrow.Schema; optional
-            The schema of the table.
-        mode: str; default "create"
-            The mode to use when creating the table. Can be either "create" or "overwrite".
-            By default, if the table already exists, an exception is raised.
-            If you want to overwrite the table, use mode="overwrite".
-        on_bad_vectors: str, default "error"
-            What to do if any of the vectors are not the same size or contains NaNs.
-            One of "error", "drop", "fill".
-        fill_value: float
-            The value to use when filling vectors. Only used if on_bad_vectors="fill".
-
-        Note
-        ----
-        The vector index won't be created by default.
-        To create the index, call the `create_index` method on the table.
-
-        Returns
-        -------
-        LanceTable
-            A reference to the newly created table.
-
-        Examples
-        --------
-
-        Can create with list of tuples or dictionaries:
-
-        >>> import lancedb
-        >>> db = lancedb.connect("./.lancedb")
-        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
-        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]
-        >>> db.create_table("my_table", data)
-        LanceTable(my_table)
-        >>> db["my_table"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        You can also pass a pandas DataFrame:
-
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({
-        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],
-        ...    "lat": [45.5, 40.1],
-        ...    "long": [-122.7, -74.1]
-        ... })
-        >>> db.create_table("table2", data)
-        LanceTable(table2)
-        >>> db["table2"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: double
-        long: double
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
-
-        Data is converted to Arrow before being written to disk. For maximum
-        control over how data is saved, either provide the PyArrow schema to
-        convert to or else provide a PyArrow table directly.
-
-        >>> custom_schema = pa.schema([
-        ...   pa.field("vector", pa.list_(pa.float32(), 2)),
-        ...   pa.field("lat", pa.float32()),
-        ...   pa.field("long", pa.float32())
-        ... ])
-        >>> db.create_table("table3", data, schema = custom_schema)
-        LanceTable(table3)
-        >>> db["table3"].head()
-        pyarrow.Table
-        vector: fixed_size_list<item: float>[2]
-          child 0, item: float
-        lat: float
-        long: float
-        ----
-        vector: [[[1.1,1.2],[0.2,1.8]]]
-        lat: [[45.5,40.1]]
-        long: [[-122.7,-74.1]]
+        See
+        ---
+        DBConnection.create_table
         """
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
 
-        if data is not None:
-            tbl = LanceTable.create(
-                self,
-                name,
-                data,
-                schema,
-                mode=mode,
-                on_bad_vectors=on_bad_vectors,
-                fill_value=fill_value,
-            )
-        else:
-            tbl = LanceTable.open(self, name)
+        tbl = LanceTable.create(
+            self,
+            name,
+            data,
+            schema,
+            mode=mode,
+            on_bad_vectors=on_bad_vectors,
+            fill_value=fill_value,
+        )
         return tbl
 
     def open_table(self, name: str) -> LanceTable:

python/lancedb/pydantic.py

@@ -0,0 +1,212 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Pydantic adapter for LanceDB"""
+
+from __future__ import annotations
+
+import inspect
+import sys
+import types
+from abc import ABC, abstractmethod
+from typing import Any, List, Type, Union, _GenericAlias
+
+import pyarrow as pa
+import pydantic
+from pydantic_core import CoreSchema, core_schema
+
+
+class FixedSizeListMixin(ABC):
+    @staticmethod
+    @abstractmethod
+    def dim() -> int:
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def value_arrow_type() -> pa.DataType:
+        raise NotImplementedError
+
+
+def vector(
+    dim: int, value_type: pa.DataType = pa.float32()
+) -> Type[FixedSizeListMixin]:
+    """Pydantic Vector Type.
+
+    !!! warning
+        Experimental feature.
+
+    Parameters
+    ----------
+    dim : int
+        The dimension of the vector.
+    value_type : pyarrow.DataType, optional
+        The value type of the vector, by default pa.float32()
+
+    Examples
+    --------
+
+    >>> import pydantic
+    >>> from lancedb.pydantic import vector
+    ...
+    >>> class MyModel(pydantic.BaseModel):
+    ...     id: int
+    ...     url: str
+    ...     embeddings: vector(768)
+    >>> schema = pydantic_to_schema(MyModel)
+    >>> assert schema == pa.schema([
+    ...     pa.field("id", pa.int64(), False),
+    ...     pa.field("url", pa.utf8(), False),
+    ...     pa.field("embeddings", pa.list_(pa.float32(), 768), False)
+    ... ])
+    """
+
+    # TODO: make a public parameterized type.
+    class FixedSizeList(list, FixedSizeListMixin):
+        @staticmethod
+        def dim() -> int:
+            return dim
+
+        @staticmethod
+        def value_arrow_type() -> pa.DataType:
+            return value_type
+
+        @classmethod
+        def __get_pydantic_core_schema__(
+            cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler
+        ) -> CoreSchema:
+            return core_schema.no_info_after_validator_function(
+                cls,
+                core_schema.list_schema(
+                    min_length=dim,
+                    max_length=dim,
+                    items_schema=core_schema.float_schema(),
+                ),
+            )
+
+    return FixedSizeList
+
+
+def _py_type_to_arrow_type(py_type: Type[Any]) -> pa.DataType:
+    """Convert Python Type to Arrow DataType.
+
+    Raises
+    ------
+    TypeError
+        If the type is not supported.
+    """
+    if py_type == int:
+        return pa.int64()
+    elif py_type == float:
+        return pa.float64()
+    elif py_type == str:
+        return pa.utf8()
+    elif py_type == bool:
+        return pa.bool_()
+    elif py_type == bytes:
+        return pa.binary()
+    raise TypeError(
+        f"Converting Pydantic type to Arrow Type: unsupported type {py_type}"
+    )
+
+
+def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
+    fields = []
+    for name, field in model.model_fields.items():
+        fields.append(_pydantic_to_field(name, field))
+    return fields
+
+
+def _pydantic_to_arrow_type(field: pydantic.fields.FieldInfo) -> pa.DataType:
+    """Convert a Pydantic FieldInfo to Arrow DataType"""
+    if isinstance(field.annotation, _GenericAlias) or (
+        sys.version_info > (3, 9) and isinstance(field.annotation, types.GenericAlias)
+    ):
+        origin = field.annotation.__origin__
+        args = field.annotation.__args__
+        if origin == list:
+            child = args[0]
+            return pa.list_(_py_type_to_arrow_type(child))
+        elif origin == Union:
+            if len(args) == 2 and args[1] == type(None):
+                return _py_type_to_arrow_type(args[0])
+    elif inspect.isclass(field.annotation):
+        if issubclass(field.annotation, pydantic.BaseModel):
+            # Struct
+            fields = _pydantic_model_to_fields(field.annotation)
+            return pa.struct(fields)
+        elif issubclass(field.annotation, FixedSizeListMixin):
+            return pa.list_(field.annotation.value_arrow_type(), field.annotation.dim())
+    return _py_type_to_arrow_type(field.annotation)
+
+
+def is_nullable(field: pydantic.fields.FieldInfo) -> bool:
+    """Check if a Pydantic FieldInfo is nullable."""
+    if isinstance(field.annotation, _GenericAlias):
+        origin = field.annotation.__origin__
+        args = field.annotation.__args__
+        if origin == Union:
+            if len(args) == 2 and args[1] == type(None):
+                return True
+    return False
+
+
+def _pydantic_to_field(name: str, field: pydantic.fields.FieldInfo) -> pa.Field:
+    """Convert a Pydantic field to a PyArrow Field."""
+    dt = _pydantic_to_arrow_type(field)
+    return pa.field(name, dt, is_nullable(field))
+
+
+def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema:
+    """Convert a Pydantic model to a PyArrow Schema.
+
+    Parameters
+    ----------
+    model : Type[pydantic.BaseModel]
+        The Pydantic BaseModel to convert to Arrow Schema.
+
+    Returns
+    -------
+    pyarrow.Schema
+
+    Examples
+    --------
+
+    >>> from typing import List, Optional
+    >>> import pydantic
+    >>> from lancedb.pydantic import pydantic_to_schema
+    ...
+    >>> class InnerModel(pydantic.BaseModel):
+    ...     a: str
+    ...     b: Optional[float]
+    >>>
+    >>> class FooModel(pydantic.BaseModel):
+    ...     id: int
+    ...     s: Optional[str] = None
+    ...     vec: List[float]
+    ...     li: List[int]
+    ...     inner: InnerModel
+    >>> schema = pydantic_to_schema(FooModel)
+    >>> assert schema == pa.schema([
+    ...     pa.field("id", pa.int64(), False),
+    ...     pa.field("s", pa.utf8(), True),
+    ...     pa.field("vec", pa.list_(pa.float64()), False),
+    ...     pa.field("li", pa.list_(pa.int64()), False),
+    ...     pa.field("inner", pa.struct([
+    ...         pa.field("a", pa.utf8(), False),
+    ...         pa.field("b", pa.float64(), True),
+    ...     ]), False),
+    ... ])
+    """
+    fields = _pydantic_model_to_fields(model)
+    return pa.schema(fields)

python/lancedb/query.py

@@ -226,6 +226,7 @@ class LanceQueryBuilder:
             columns=self._columns,
             nprobes=self._nprobes,
             refine_factor=self._refine_factor,
+            vector_column=self._vector_column,
         )
         return self._table._execute_query(query)
 

python/lancedb/remote/arrow.py

@@ -0,0 +1,22 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import pyarrow as pa
+
+
+def to_ipc_binary(table: pa.Table) -> bytes:
+    """Serialize a PyArrow Table to IPC binary."""
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()

python/lancedb/remote/client.py

@@ -13,16 +13,19 @@
 
 
 import functools
-from typing import Dict
+from typing import Any, Callable, Dict, Optional, Union
 
 import aiohttp
 import attr
 import pyarrow as pa
+from pydantic import BaseModel
 
 from lancedb.common import Credential
 from lancedb.remote import VectorQuery, VectorQueryResult
 from lancedb.remote.errors import LanceDBClientError
 
+ARROW_STREAM_CONTENT_TYPE = "application/vnd.apache.arrow.stream"
+
 
 def _check_not_closed(f):
     @functools.wraps(f)
@@ -34,6 +37,12 @@ def _check_not_closed(f):
     return wrapped
 
 
+async def _read_ipc(resp: aiohttp.ClientResponse) -> pa.Table:
+    resp_body = await resp.read()
+    with pa.ipc.open_file(pa.BufferReader(resp_body)) as reader:
+        return reader.read_all()
+
+
 @attr.define(slots=False)
 class RestfulLanceDBClient:
     db_name: str
@@ -52,32 +61,85 @@ class RestfulLanceDBClient:
 
     @functools.cached_property
     def headers(self) -> Dict[str, str]:
-        return {
+        headers = {
             "x-api-key": self.api_key,
         }
+        if self.region == "local":  # Local test mode
+            headers["Host"] = f"{self.db_name}.{self.region}.api.lancedb.com"
+        return headers
+
+    @staticmethod
+    async def _check_status(resp: aiohttp.ClientResponse):
+        if resp.status == 404:
+            raise LanceDBClientError(f"Not found: {await resp.text()}")
+        elif 400 <= resp.status < 500:
+            raise LanceDBClientError(
+                f"Bad Request: {resp.status}, error: {await resp.text()}"
+            )
+        elif 500 <= resp.status < 600:
+            raise LanceDBClientError(
+                f"Internal Server Error: {resp.status}, error: {await resp.text()}"
+            )
+        elif resp.status != 200:
+            raise LanceDBClientError(
+                f"Unknown Error: {resp.status}, error: {await resp.text()}"
+            )
+
+    @_check_not_closed
+    async def get(self, uri: str, params: Union[Dict[str, Any], BaseModel] = None):
+        """Send a GET request and returns the deserialized response payload."""
+        if isinstance(params, BaseModel):
+            params: Dict[str, Any] = params.dict(exclude_none=True)
+        async with self.session.get(uri, params=params, headers=self.headers) as resp:
+            await self._check_status(resp)
+            return await resp.json()
+
+    @_check_not_closed
+    async def post(
+        self,
+        uri: str,
+        data: Union[Dict[str, Any], BaseModel, bytes],
+        params: Optional[Dict[str, Any]] = None,
+        content_type: Optional[str] = None,
+        deserialize: Callable = lambda resp: resp.json(),
+    ) -> Dict[str, Any]:
+        """Send a POST request and returns the deserialized response payload.
+
+        Parameters
+        ----------
+        uri : str
+            The uri to send the POST request to.
+        data: Union[Dict[str, Any], BaseModel]
+
+        """
+        if isinstance(data, BaseModel):
+            data: Dict[str, Any] = data.dict(exclude_none=True)
+        if isinstance(data, bytes):
+            req_kwargs = {"data": data}
+        else:
+            req_kwargs = {"json": data}
+
+        headers = self.headers.copy()
+        if content_type is not None:
+            headers["content-type"] = content_type
+        async with self.session.post(
+            uri,
+            headers=headers,
+            params=params,
+            **req_kwargs,
+        ) as resp:
+            resp: aiohttp.ClientResponse = resp
+            await self._check_status(resp)
+            return await deserialize(resp)
+
+    @_check_not_closed
+    async def list_tables(self):
+        """List all tables in the database."""
+        json = await self.get("/v1/table/", {})
+        return json["tables"]
 
     @_check_not_closed
     async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
-        async with self.session.post(
-            f"/1/table/{table_name}/",
-            json=query.dict(exclude_none=True),
-            headers=self.headers,
-        ) as resp:
-            resp: aiohttp.ClientResponse = resp
-            if 400 <= resp.status < 500:
-                raise LanceDBClientError(
-                    f"Bad Request: {resp.status}, error: {await resp.text()}"
-                )
-            if 500 <= resp.status < 600:
-                raise LanceDBClientError(
-                    f"Internal Server Error: {resp.status}, error: {await resp.text()}"
-                )
-            if resp.status != 200:
-                raise LanceDBClientError(
-                    f"Unknown Error: {resp.status}, error: {await resp.text()}"
-                )
-
-            resp_body = await resp.read()
-            with pa.ipc.open_file(pa.BufferReader(resp_body)) as reader:
-                tbl = reader.read_all()
+        """Query a table."""
+        tbl = await self.post(f"/v1/table/{table_name}/", query, deserialize=_read_ipc)
         return VectorQueryResult(tbl)

python/lancedb/remote/db.py

@@ -11,6 +11,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import asyncio
+import uuid
 from typing import List
 from urllib.parse import urlparse
 
@@ -18,9 +20,11 @@ import pyarrow as pa
 
 from lancedb.common import DATA
 from lancedb.db import DBConnection
-from lancedb.table import Table
+from lancedb.schema import schema_to_json
+from lancedb.table import Table, _sanitize_data
 
-from .client import RestfulLanceDBClient
+from .arrow import to_ipc_binary
+from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
 
 
 class RemoteDBConnection(DBConnection):
@@ -34,12 +38,18 @@ class RemoteDBConnection(DBConnection):
         self.db_name = parsed.netloc
         self.api_key = api_key
         self._client = RestfulLanceDBClient(self.db_name, region, api_key)
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = asyncio.get_event_loop()
 
     def __repr__(self) -> str:
         return f"RemoveConnect(name={self.db_name})"
 
     def table_names(self) -> List[str]:
-        raise NotImplementedError
+        """List the names of all tables in the database."""
+        result = self._loop.run_until_complete(self._client.list_tables())
+        return result
 
     def open_table(self, name: str) -> Table:
         """Open a Lance Table in the database.
@@ -64,8 +74,31 @@ class RemoteDBConnection(DBConnection):
         name: str,
         data: DATA = None,
         schema: pa.Schema = None,
-        mode: str = "create",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> Table:
-        raise NotImplementedError
+        if data is None and schema is None:
+            raise ValueError("Either data or schema must be provided.")
+        if data is not None:
+            data = _sanitize_data(
+                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            )
+        else:
+            if schema is None:
+                raise ValueError("Either data or schema must be provided")
+            data = pa.Table.from_pylist([], schema=schema)
+
+        from .table import RemoteTable
+
+        data = to_ipc_binary(data)
+        request_id = uuid.uuid4().hex
+
+        self._loop.run_until_complete(
+            self._client.post(
+                f"/v1/table/{name}/create",
+                data=data,
+                params={"request_id": request_id},
+                content_type=ARROW_STREAM_CONTENT_TYPE,
+            )
+        )
+        return RemoteTable(self, name)

python/lancedb/remote/table.py

@@ -11,7 +11,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import asyncio
+import uuid
+from functools import cached_property
 from typing import Union
 
 import pyarrow as pa
@@ -19,7 +20,10 @@ import pyarrow as pa
 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
 from ..query import LanceQueryBuilder, Query
-from ..table import Query, Table
+from ..schema import json_to_schema
+from ..table import Query, Table, _sanitize_data
+from .arrow import to_ipc_binary
+from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
 
 
@@ -31,8 +35,14 @@ class RemoteTable(Table):
     def __repr__(self) -> str:
         return f"RemoteTable({self._conn.db_name}.{self.name})"
 
+    @cached_property
     def schema(self) -> pa.Schema:
-        raise NotImplementedError
+        """Return the schema of the table."""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.get(f"/v1/table/{self._name}/describe")
+        )
+        schema = json_to_schema(resp["schema"])
+        return schema
 
     def to_arrow(self) -> pa.Table:
         raise NotImplementedError
@@ -54,7 +64,21 @@ class RemoteTable(Table):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
     ) -> int:
-        raise NotImplementedError
+        data = _sanitize_data(
+            data, self.schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+        )
+        payload = to_ipc_binary(data)
+
+        request_id = uuid.uuid4().hex
+
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(
+                f"/v1/table/{self._name}/insert",
+                data=payload,
+                params={"request_id": request_id, "mode": mode},
+                content_type=ARROW_STREAM_CONTENT_TYPE,
+            )
+        )
 
     def search(
         self, query: Union[VEC, str], vector_column: str = VECTOR_COLUMN_NAME
@@ -62,9 +86,8 @@ class RemoteTable(Table):
         return LanceQueryBuilder(self, query, vector_column)
 
     def _execute_query(self, query: Query) -> pa.Table:
-        try:
-            loop = asyncio.get_running_loop()
-        except RuntimeError:
-            loop = asyncio.get_event_loop()
         result = self._conn._client.query(self._name, query)
-        return loop.run_until_complete(result).to_arrow()
+        return self._conn._loop.run_until_complete(result).to_arrow()
+
+    def delete(self, predicate: str):
+        raise NotImplementedError

python/lancedb/schema.py

@@ -0,0 +1,45 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Schema related utilities."""
+
+from typing import Any, Dict, Type
+
+import pyarrow as pa
+from lance import json_to_schema, schema_to_json
+
+
+def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
+    """A help function to create a vector type.
+
+    Parameters
+    ----------
+    dimension: The dimension of the vector.
+    value_type: pa.DataType, optional
+        The type of the value in the vector.
+
+    Returns
+    -------
+    A PyArrow DataType for vectors.
+
+    Examples
+    --------
+
+    >>> import pyarrow as pa
+    >>> import lancedb
+    >>> schema = pa.schema([
+    ...     pa.field("id", pa.int64()),
+    ...     pa.field("vector", lancedb.vector(756)),
+    ... ])
+    """
+    return pa.list_(value_type, dimension)

python/lancedb/table.py

@@ -16,7 +16,7 @@ from __future__ import annotations
 import os
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import List, Union
+from typing import Iterable, List, Union
 
 import lance
 import numpy as np
@@ -44,7 +44,7 @@ def _sanitize_data(data, schema, on_bad_vectors, fill_value):
         data = _sanitize_schema(
             data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
-    if not isinstance(data, pa.Table):
+    if not isinstance(data, (pa.Table, Iterable)):
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
 
@@ -74,7 +74,6 @@ class Table(ABC):
     Can append new data with [Table.add()][lancedb.table.Table.add].
 
     >>> table.add([{"vector": [0.5, 1.3], "b": 4}])
-    2
 
     Can query the table with [Table.search][lancedb.table.Table.search].
 
@@ -151,7 +150,7 @@ class Table(ABC):
         mode: str = "append",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-    ) -> int:
+    ):
         """Add more data to the [Table](Table).
 
         Parameters
@@ -167,10 +166,6 @@ class Table(ABC):
         fill_value: float, default 0.
             The value to use when filling vectors. Only used if on_bad_vectors="fill".
 
-        Returns
-        -------
-        int
-            The number of vectors in the table.
         """
         raise NotImplementedError
 
@@ -202,6 +197,51 @@ class Table(ABC):
     def _execute_query(self, query: Query) -> pa.Table:
         pass
 
+    @abstractmethod
+    def delete(self, where: str):
+        """Delete rows from the table.
+
+        This can be used to delete a single row, many rows, all rows, or
+        sometimes no rows (if your predicate matches nothing).
+
+        Parameters
+        ----------
+        where: str
+            The SQL where clause to use when deleting rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
+        >>> db = lancedb.connect("./.lancedb")
+        >>> table = db.create_table("my_table", data)
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  2  [3.0, 4.0]
+        2  3  [5.0, 6.0]
+        >>> table.delete("x = 2")
+        >>> table.to_pandas()
+           x      vector
+        0  1  [1.0, 2.0]
+        1  3  [5.0, 6.0]
+
+        If you have a list of values to delete, you can combine them into a
+        stringified list and use the `IN` operator:
+
+        >>> to_remove = [1, 5]
+        >>> to_remove = ", ".join([str(v) for v in to_remove])
+        >>> to_remove
+        '1, 5'
+        >>> table.delete(f"x IN ({to_remove})")
+        >>> table.to_pandas()
+           x      vector
+        0  3  [5.0, 6.0]
+        """
+        raise NotImplementedError
+
 
 class LanceTable(Table):
     """
@@ -262,7 +302,6 @@ class LanceTable(Table):
                vector    type
         0  [1.1, 0.9]  vector
         >>> table.add([{"vector": [0.5, 0.2], "type": "vector"}])
-        2
         >>> table.version
         2
         >>> table.checkout(1)
@@ -364,7 +403,7 @@ class LanceTable(Table):
         mode: str = "append",
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
-    ) -> int:
+    ):
         """Add data to the table.
 
         Parameters
@@ -391,7 +430,6 @@ class LanceTable(Table):
         )
         lance.write_dataset(data, self._dataset_uri, mode=mode)
         self._reset_dataset()
-        return len(self)
 
     def search(
         self, query: Union[VEC, str], vector_column_name=VECTOR_COLUMN_NAME
@@ -483,44 +521,21 @@ class LanceTable(Table):
             if schema is None:
                 raise ValueError("Either data or schema must be provided")
             data = pa.Table.from_pylist([], schema=schema)
-        lance.write_dataset(data, tbl._dataset_uri, mode=mode)
+        lance.write_dataset(data, tbl._dataset_uri, schema=schema, mode=mode)
         return LanceTable(db, name)
 
     @classmethod
     def open(cls, db, name):
         tbl = cls(db, name)
-        if not os.path.exists(tbl._dataset_uri):
+        fs, path = pa.fs.FileSystem.from_uri(tbl._dataset_uri)
+        file_info = fs.get_file_info(path)
+        if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(
                 f"Table {name} does not exist. Please first call db.create_table({name}, data)"
             )
         return tbl
 
     def delete(self, where: str):
-        """Delete rows from the table.
-
-        Parameters
-        ----------
-        where: str
-            The SQL where clause to use when deleting rows.
-
-        Examples
-        --------
-        >>> import lancedb
-        >>> import pandas as pd
-        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
-        >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", data)
-        >>> table.to_pandas()
-           x      vector
-        0  1  [1.0, 2.0]
-        1  2  [3.0, 4.0]
-        2  3  [5.0, 6.0]
-        >>> table.delete("x = 2")
-        >>> table.to_pandas()
-           x      vector
-        0  1  [1.0, 2.0]
-        1  3  [5.0, 6.0]
-        """
         self._dataset.delete(where)
 
     def _execute_query(self, query: Query) -> pa.Table:

python/lancedb/util.py

@@ -11,8 +11,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import os
+from typing import Tuple
 from urllib.parse import urlparse
 
+import pyarrow as pa
+import pyarrow.fs as pa_fs
+
 
 def get_uri_scheme(uri: str) -> str:
     """
@@ -59,3 +64,14 @@ def get_uri_location(uri: str) -> str:
         return parsed.path
     else:
         return parsed.netloc + parsed.path
+
+
+def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
+    """
+    Get a PyArrow FileSystem from a URI, handling extra environment variables.
+    """
+    if get_uri_scheme(uri) == "s3":
+        if os.environ["AWS_ENDPOINT"]:
+            uri += "?endpoint_override=" + os.environ["AWS_ENDPOINT"]
+
+    return pa_fs.FileSystem.from_uri(uri)

python/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lancedb"
-version = "0.1.10"
-dependencies = ["pylance~=0.5.0", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic", "attr"]
+version = "0.1.12"
+dependencies = ["pylance~=0.5.8", "ratelimiter", "retry", "tqdm", "aiohttp", "pydantic>=2", "attr"]
 description = "lancedb"
 authors = [
     { name = "LanceDB Devs", email = "dev@lancedb.com" },

python/tests/test_db.py

@@ -13,6 +13,7 @@
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
 import pytest
 
 import lancedb
@@ -75,6 +76,32 @@ def test_ingest_pd(tmp_path):
     assert db.open_table("test").name == db["test"].name
 
 
+def test_ingest_record_batch_iterator(tmp_path):
+    def batch_reader():
+        for i in range(5):
+            yield pa.RecordBatch.from_arrays(
+                [
+                    pa.array([[3.1, 4.1], [5.9, 26.5]]),
+                    pa.array(["foo", "bar"]),
+                    pa.array([10.0, 20.0]),
+                ],
+                ["vector", "item", "price"],
+            )
+
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table(
+        "test",
+        batch_reader(),
+        schema=pa.schema(
+            [
+                pa.field("vector", pa.list_(pa.float32())),
+                pa.field("item", pa.utf8()),
+                pa.field("price", pa.float32()),
+            ]
+        ),
+    )
+
+
 def test_create_mode(tmp_path):
     db = lancedb.connect(tmp_path)
     data = pd.DataFrame(
@@ -131,6 +158,9 @@ def test_empty_or_nonexistent_table(tmp_path):
     with pytest.raises(Exception):
         db.open_table("does_not_exist")
 
+    schema = pa.schema([pa.field("a", pa.int32())])
+    db.create_table("test", schema=schema)
+
 
 def test_replace_index(tmp_path):
     db = lancedb.connect(uri=tmp_path)

python/tests/test_pydantic.py

@@ -0,0 +1,155 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+import json
+import sys
+from typing import List, Optional
+
+import pyarrow as pa
+import pydantic
+import pytest
+
+from lancedb.pydantic import pydantic_to_schema, vector
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 9),
+    reason="using native type alias requires python3.9 or higher",
+)
+def test_pydantic_to_arrow():
+    class StructModel(pydantic.BaseModel):
+        a: str
+        b: Optional[float]
+
+    class TestModel(pydantic.BaseModel):
+        id: int
+        s: str
+        vec: list[float]
+        li: List[int]
+        opt: Optional[str] = None
+        st: StructModel
+        # d: dict
+
+    m = TestModel(
+        id=1, s="hello", vec=[1.0, 2.0, 3.0], li=[2, 3, 4], st=StructModel(a="a", b=1.0)
+    )
+
+    schema = pydantic_to_schema(TestModel)
+
+    expect_schema = pa.schema(
+        [
+            pa.field("id", pa.int64(), False),
+            pa.field("s", pa.utf8(), False),
+            pa.field("vec", pa.list_(pa.float64()), False),
+            pa.field("li", pa.list_(pa.int64()), False),
+            pa.field("opt", pa.utf8(), True),
+            pa.field(
+                "st",
+                pa.struct(
+                    [pa.field("a", pa.utf8(), False), pa.field("b", pa.float64(), True)]
+                ),
+                False,
+            ),
+        ]
+    )
+    assert schema == expect_schema
+
+
+def test_pydantic_to_arrow_py38():
+    class StructModel(pydantic.BaseModel):
+        a: str
+        b: Optional[float]
+
+    class TestModel(pydantic.BaseModel):
+        id: int
+        s: str
+        vec: List[float]
+        li: List[int]
+        opt: Optional[str] = None
+        st: StructModel
+        # d: dict
+
+    m = TestModel(
+        id=1, s="hello", vec=[1.0, 2.0, 3.0], li=[2, 3, 4], st=StructModel(a="a", b=1.0)
+    )
+
+    schema = pydantic_to_schema(TestModel)
+
+    expect_schema = pa.schema(
+        [
+            pa.field("id", pa.int64(), False),
+            pa.field("s", pa.utf8(), False),
+            pa.field("vec", pa.list_(pa.float64()), False),
+            pa.field("li", pa.list_(pa.int64()), False),
+            pa.field("opt", pa.utf8(), True),
+            pa.field(
+                "st",
+                pa.struct(
+                    [pa.field("a", pa.utf8(), False), pa.field("b", pa.float64(), True)]
+                ),
+                False,
+            ),
+        ]
+    )
+    assert schema == expect_schema
+
+
+def test_fixed_size_list_field():
+    class TestModel(pydantic.BaseModel):
+        vec: vector(16)
+        li: List[int]
+
+    data = TestModel(vec=list(range(16)), li=[1, 2, 3])
+    assert json.loads(data.model_dump_json()) == {
+        "vec": list(range(16)),
+        "li": [1, 2, 3],
+    }
+
+    schema = pydantic_to_schema(TestModel)
+    assert schema == pa.schema(
+        [
+            pa.field("vec", pa.list_(pa.float32(), 16), False),
+            pa.field("li", pa.list_(pa.int64()), False),
+        ]
+    )
+
+    json_schema = TestModel.model_json_schema()
+    assert json_schema == {
+        "properties": {
+            "vec": {
+                "items": {"type": "number"},
+                "maxItems": 16,
+                "minItems": 16,
+                "title": "Vec",
+                "type": "array",
+            },
+            "li": {"items": {"type": "integer"}, "title": "Li", "type": "array"},
+        },
+        "required": ["vec", "li"],
+        "title": "TestModel",
+        "type": "object",
+    }
+
+
+def test_fixed_size_list_validation():
+    class TestModel(pydantic.BaseModel):
+        vec: vector(8)
+
+    with pytest.raises(pydantic.ValidationError):
+        TestModel(vec=range(9))
+
+    with pytest.raises(pydantic.ValidationError):
+        TestModel(vec=range(7))
+
+    TestModel(vec=range(8))

python/tests/test_query.py

@@ -119,6 +119,7 @@ def test_query_builder_with_different_vector_column():
             columns=["b"],
             nprobes=20,
             refine_factor=None,
+            vector_column="foo_vector",
         )
     )
 

python/tests/test_table.py

@@ -139,8 +139,8 @@ def _add(table, schema):
     # table = LanceTable(db, "test")
     assert len(table) == 2
 
-    count = table.add([{"vector": [6.3, 100.5], "item": "new", "price": 30.0}])
-    assert count == 3
+    table.add([{"vector": [6.3, 100.5], "item": "new", "price": 30.0}])
+    assert len(table) == 3
 
     expected = pa.Table.from_arrays(
         [

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.10"
+version = "0.1.14"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -15,7 +15,11 @@ arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
 once_cell = "1"
 futures = "0.3"
+half = { workspace = true }
 lance = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }
+object_store = { workspace = true, features = ["aws"] }
+async-trait = "0"
+env_logger = "0"

rust/ffi/node/src/arrow.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::io::Cursor;
-use std::ops::Deref;
 use std::sync::Arc;
 
 use arrow_array::cast::as_list_array;
@@ -25,10 +24,13 @@ use lance::arrow::{FixedSizeListArrayExt, RecordBatchExt};
 pub(crate) fn convert_record_batch(record_batch: RecordBatch) -> RecordBatch {
     let column = record_batch
         .column_by_name("vector")
+        .cloned()
         .expect("vector column is missing");
-    let arr = as_list_array(column.deref());
+    // TODO: we should just consume the underlaying js buffer in the future instead of this arrow around a bunch of times
+    let arr = as_list_array(column.as_ref());
     let list_size = arr.values().len() / record_batch.num_rows();
-    let r = FixedSizeListArray::try_new(arr.values(), list_size as i32).unwrap();
+    let r =
+        FixedSizeListArray::try_new_from_values(arr.values().to_owned(), list_size as i32).unwrap();
 
     let schema = Arc::new(Schema::new(vec![Field::new(
         "vector",

rust/ffi/node/src/lib.rs

@@ -17,19 +17,23 @@ use std::convert::TryFrom;
 use std::ops::Deref;
 use std::sync::{Arc, Mutex};
 
-use arrow_array::{Float32Array, RecordBatchIterator, RecordBatchReader};
+use arrow_array::{Float32Array, RecordBatchIterator};
 use arrow_ipc::writer::FileWriter;
+use async_trait::async_trait;
 use futures::{TryFutureExt, TryStreamExt};
 use lance::dataset::{WriteMode, WriteParams};
 use lance::index::vector::MetricType;
+use lance::io::object_store::ObjectStoreParams;
 use neon::prelude::*;
 use neon::types::buffer::TypedArray;
+use object_store::aws::{AwsCredential, AwsCredentialProvider};
+use object_store::CredentialProvider;
 use once_cell::sync::OnceCell;
 use tokio::runtime::Runtime;
 
 use vectordb::database::Database;
 use vectordb::error::Error;
-use vectordb::table::Table;
+use vectordb::table::{ReadParams, Table};
 
 use crate::arrow::arrow_buffer_to_record_batch;
 
@@ -49,8 +53,38 @@ struct JsTable {
 
 impl Finalize for JsTable {}
 
+// TODO: object_store didn't export this type so I copied it.
+// Make a requiest to object_store to export this type
+#[derive(Debug)]
+pub struct StaticCredentialProvider<T> {
+    credential: Arc<T>,
+}
+
+impl<T> StaticCredentialProvider<T> {
+    pub fn new(credential: T) -> Self {
+        Self {
+            credential: Arc::new(credential),
+        }
+    }
+}
+
+#[async_trait]
+impl<T> CredentialProvider for StaticCredentialProvider<T>
+where
+    T: std::fmt::Debug + Send + Sync,
+{
+    type Credential = T;
+
+    async fn get_credential(&self) -> object_store::Result<Arc<T>> {
+        Ok(Arc::clone(&self.credential))
+    }
+}
+
 fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
     static RUNTIME: OnceCell<Runtime> = OnceCell::new();
+    static LOG: OnceCell<()> = OnceCell::new();
+
+    LOG.get_or_init(|| env_logger::init());
 
     RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
 }
@@ -97,19 +131,67 @@ fn database_table_names(mut cx: FunctionContext) -> JsResult<JsPromise> {
     Ok(promise)
 }
 
+fn get_aws_creds<T>(
+    cx: &mut FunctionContext,
+    arg_starting_location: i32,
+) -> Result<Option<AwsCredentialProvider>, NeonResult<T>> {
+    let secret_key_id = cx
+        .argument_opt(arg_starting_location)
+        .map(|arg| arg.downcast_or_throw::<JsString, FunctionContext>(cx).ok())
+        .flatten()
+        .map(|v| v.value(cx));
+
+    let secret_key = cx
+        .argument_opt(arg_starting_location + 1)
+        .map(|arg| arg.downcast_or_throw::<JsString, FunctionContext>(cx).ok())
+        .flatten()
+        .map(|v| v.value(cx));
+
+    let temp_token = cx
+        .argument_opt(arg_starting_location + 2)
+        .map(|arg| arg.downcast_or_throw::<JsString, FunctionContext>(cx).ok())
+        .flatten()
+        .map(|v| v.value(cx));
+
+    match (secret_key_id, secret_key, temp_token) {
+        (Some(key_id), Some(key), optional_token) => Ok(Some(Arc::new(
+            StaticCredentialProvider::new(AwsCredential {
+                key_id: key_id,
+                secret_key: key,
+                token: optional_token,
+            }),
+        ))),
+        (None, None, None) => Ok(None),
+        _ => Err(cx.throw_error("Invalid credentials configuration")),
+    }
+}
+
 fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let db = cx
         .this()
         .downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
     let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
 
+    let aws_creds = match get_aws_creds(&mut cx, 1) {
+        Ok(creds) => creds,
+        Err(err) => return err,
+    };
+
+    let params = ReadParams {
+        store_options: Some(ObjectStoreParams {
+            aws_credentials: aws_creds,
+            ..ObjectStoreParams::default()
+        }),
+        ..ReadParams::default()
+    };
+
     let rt = runtime(&mut cx)?;
     let channel = cx.channel();
     let database = db.database.clone();
 
     let (deferred, promise) = cx.promise();
     rt.spawn(async move {
-        let table_rst = database.open_table(&table_name).await;
+        let table_rst = database.open_table_with_params(&table_name, &params).await;
 
         deferred.settle_with(&channel, move |mut cx| {
             let table = Arc::new(Mutex::new(
@@ -241,8 +323,6 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
         "create" => WriteMode::Create,
         _ => return cx.throw_error("Table::create only supports 'overwrite' and 'create' modes"),
     };
-    let mut params = WriteParams::default();
-    params.mode = mode;
 
     let rt = runtime(&mut cx)?;
     let channel = cx.channel();
@@ -250,11 +330,22 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let (deferred, promise) = cx.promise();
     let database = db.database.clone();
 
+    let aws_creds = match get_aws_creds(&mut cx, 3) {
+        Ok(creds) => creds,
+        Err(err) => return err,
+    };
+
+    let params = WriteParams {
+        store_params: Some(ObjectStoreParams {
+            aws_credentials: aws_creds,
+            ..ObjectStoreParams::default()
+        }),
+        mode: mode,
+        ..WriteParams::default()
+    };
+
     rt.block_on(async move {
-        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
-            batches.into_iter().map(Ok),
-            schema,
-        ));
+        let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
         let table_rst = database
             .create_table(&table_name, batch_reader, Some(params))
             .await;
@@ -289,16 +380,27 @@ fn table_add(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let table = js_table.table.clone();
     let write_mode = write_mode_map.get(write_mode.as_str()).cloned();
 
+    let aws_creds = match get_aws_creds(&mut cx, 2) {
+        Ok(creds) => creds,
+        Err(err) => return err,
+    };
+
+    let params = WriteParams {
+        store_params: Some(ObjectStoreParams {
+            aws_credentials: aws_creds,
+            ..ObjectStoreParams::default()
+        }),
+        mode: write_mode.unwrap_or(WriteMode::Append),
+        ..WriteParams::default()
+    };
+
     rt.block_on(async move {
-        let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
-            batches.into_iter().map(Ok),
-            schema,
-        ));
-        let add_result = table.lock().unwrap().add(batch_reader, write_mode).await;
+        let batch_reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        let add_result = table.lock().unwrap().add(batch_reader, Some(params)).await;
 
         deferred.settle_with(&channel, move |mut cx| {
-            let added = add_result.or_else(|err| cx.throw_error(err.to_string()))?;
-            Ok(cx.number(added as f64))
+            let _added = add_result.or_else(|err| cx.throw_error(err.to_string()))?;
+            Ok(cx.boolean(true))
         });
     });
     Ok(promise)

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.1.10"
+version = "0.1.14"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -13,6 +13,7 @@ arrow-data = { workspace = true }
 arrow-schema = { workspace = true }
 object_store = { workspace = true }
 snafu = "0.7.4"
+half = { workspace = true }
 lance = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 

rust/vectordb/src/database.rs

@@ -20,13 +20,16 @@ use lance::dataset::WriteParams;
 use lance::io::object_store::ObjectStore;
 use snafu::prelude::*;
 
-use crate::error::{CreateDirSnafu, Result};
-use crate::table::{OpenTableParams, Table};
+use crate::error::{CreateDirSnafu, InvalidTableNameSnafu, Result};
+use crate::table::{ReadParams, Table};
+
+pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 pub struct Database {
     object_store: ObjectStore,
 
     pub(crate) uri: String,
+    pub(crate) base_path: object_store::path::Path,
 }
 
 const LANCE_EXTENSION: &str = "lance";
@@ -43,12 +46,13 @@ impl Database {
     ///
     /// * A [Database] object.
     pub async fn connect(uri: &str) -> Result<Database> {
-        let (object_store, _) = ObjectStore::from_uri(uri).await?;
+        let (object_store, base_path) = ObjectStore::from_uri(uri).await?;
         if object_store.is_local() {
             Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
         }
         Ok(Database {
             uri: uri.to_string(),
+            base_path,
             object_store,
         })
     }
@@ -57,7 +61,7 @@ impl Database {
     fn try_create_dir(path: &str) -> core::result::Result<(), std::io::Error> {
         let path = Path::new(path);
         if !path.try_exists()? {
-            create_dir_all(&path)?;
+            create_dir_all(path)?;
         }
         Ok(())
     }
@@ -70,23 +74,18 @@ impl Database {
     pub async fn table_names(&self) -> Result<Vec<String>> {
         let f = self
             .object_store
-            .read_dir(self.uri.as_str())
+            .read_dir(self.base_path.clone())
             .await?
             .iter()
-            .map(|fname| Path::new(fname))
+            .map(Path::new)
             .filter(|path| {
                 let is_lance = path
                     .extension()
-                    .map(|e| e.to_str().map(|e| e == LANCE_EXTENSION))
-                    .flatten();
+                    .and_then(|e| e.to_str())
+                    .map(|e| e == LANCE_EXTENSION);
                 is_lance.unwrap_or(false)
             })
-            .map(|p| {
-                p.file_stem()
-                    .map(|s| s.to_str().map(|s| String::from(s)))
-                    .flatten()
-            })
-            .flatten()
+            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
             .collect();
         Ok(f)
     }
@@ -100,10 +99,11 @@ impl Database {
     pub async fn create_table(
         &self,
         name: &str,
-        batches: Box<dyn RecordBatchReader>,
+        batches: impl RecordBatchReader + Send + 'static,
         params: Option<WriteParams>,
     ) -> Result<Table> {
-        Table::create(&self.uri, name, batches, params).await
+        let table_uri = self.table_uri(name)?;
+        Table::create(&table_uri, name, batches, params).await
     }
 
     /// Open a table in the database.
@@ -115,7 +115,7 @@ impl Database {
     ///
     /// * A [Table] object.
     pub async fn open_table(&self, name: &str) -> Result<Table> {
-        self.open_table_with_params(name, OpenTableParams::default())
+        self.open_table_with_params(name, &ReadParams::default())
             .await
     }
 
@@ -128,12 +128,9 @@ impl Database {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_table_with_params(
-        &self,
-        name: &str,
-        params: OpenTableParams,
-    ) -> Result<Table> {
-        Table::open_with_params(&self.uri, name, params).await
+    pub async fn open_table_with_params(&self, name: &str, params: &ReadParams) -> Result<Table> {
+        let table_uri = self.table_uri(name)?;
+        Table::open_with_params(&table_uri, name, params).await
     }
 
     /// Drop a table in the database.
@@ -141,10 +138,23 @@ impl Database {
     /// # Arguments
     /// * `name` - The name of the table.
     pub async fn drop_table(&self, name: &str) -> Result<()> {
-        let dir_name = format!("{}/{}.{}", self.uri, name, LANCE_EXTENSION);
-        self.object_store.remove_dir_all(dir_name).await?;
+        let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
+        let full_path = self.base_path.child(dir_name.clone());
+        self.object_store.remove_dir_all(full_path).await?;
         Ok(())
     }
+
+    /// Get the URI of a table in the database.
+    fn table_uri(&self, name: &str) -> Result<String> {
+        let path = Path::new(&self.uri);
+        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+
+        let uri = table_uri
+            .as_path()
+            .to_str()
+            .context(InvalidTableNameSnafu { name })?;
+        Ok(uri.to_string())
+    }
 }
 
 #[cfg(test)]

rust/vectordb/src/index/vector.rs

@@ -35,6 +35,12 @@ pub struct IvfPQIndexBuilder {
 
 impl IvfPQIndexBuilder {
     pub fn new() -> IvfPQIndexBuilder {
+        Default::default()
+    }
+}
+
+impl Default for IvfPQIndexBuilder {
+    fn default() -> Self {
         IvfPQIndexBuilder {
             column: None,
             index_name: None,

rust/vectordb/src/query.rs

@@ -173,10 +173,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_setters_getters() {
-        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
-        let ds = Dataset::write(&mut batches, "memory://foo", None)
-            .await
-            .unwrap();
+        let batches = make_test_batches();
+        let ds = Dataset::write(batches, "memory://foo", None).await.unwrap();
 
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = Query::new(Arc::new(ds), vector.clone());
@@ -202,10 +200,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_execute() {
-        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
-        let ds = Dataset::write(&mut batches, "memory://foo", None)
-            .await
-            .unwrap();
+        let batches = make_test_batches();
+        let ds = Dataset::write(batches, "memory://foo", None).await.unwrap();
 
         let vector = Float32Array::from_iter_values([0.1; 128]);
         let query = Query::new(Arc::new(ds), vector.clone());
@@ -213,7 +209,7 @@ mod tests {
         assert_eq!(result.is_ok(), true);
     }
 
-    fn make_test_batches() -> Box<dyn RecordBatchReader> {
+    fn make_test_batches() -> impl RecordBatchReader + Send + 'static {
         let dim: usize = 128;
         let schema = Arc::new(ArrowSchema::new(vec![
             ArrowField::new("key", DataType::Int32, false),
@@ -227,11 +223,11 @@ mod tests {
             ),
             ArrowField::new("uri", DataType::Utf8, true),
         ]));
-        Box::new(RecordBatchIterator::new(
+        RecordBatchIterator::new(
             vec![RecordBatch::new_empty(schema.clone())]
                 .into_iter()
                 .map(Ok),
             schema,
-        ))
+        )
     }
 }

rust/vectordb/src/table.rs

@@ -12,21 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
-use lance::dataset::{Dataset, ReadParams, WriteParams};
+use arrow_schema::SchemaRef;
+use lance::dataset::{Dataset, WriteParams};
 use lance::index::IndexType;
-use snafu::prelude::*;
+use std::path::Path;
 
-use crate::error::{Error, InvalidTableNameSnafu, Result};
+use crate::error::{Error, Result};
 use crate::index::vector::VectorIndexBuilder;
-use crate::WriteMode;
 use crate::query::Query;
+use crate::WriteMode;
+
+pub use lance::dataset::ReadParams;
 
 pub const VECTOR_COLUMN_NAME: &str = "vector";
-pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 /// A table in a LanceDB database.
 #[derive(Debug, Clone)]
@@ -42,24 +43,25 @@ impl std::fmt::Display for Table {
     }
 }
 
-#[derive(Default)]
-pub struct OpenTableParams {
-    pub open_table_params: ReadParams,
-}
-
 impl Table {
     /// Opens an existing Table
     ///
     /// # Arguments
     ///
-    /// * `base_path` - The base path where the table is located
-    /// * `name` The Table name
+    /// * `uri` - The uri to a [Table]
+    /// * `name` - The table name
     ///
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open(base_uri: &str, name: &str) -> Result<Self> {
-        Self::open_with_params(base_uri, name, OpenTableParams::default()).await
+    pub async fn open(uri: &str) -> Result<Self> {
+        let name = Self::get_table_name(uri)?;
+        Self::open_with_params(uri, &name, &ReadParams::default()).await
+    }
+
+    /// Open an Table with a given name.
+    pub async fn open_with_name(uri: &str, name: &str) -> Result<Self> {
+        Self::open_with_params(uri, name, &ReadParams::default()).await
     }
 
     /// Opens an existing Table
@@ -68,25 +70,13 @@ impl Table {
     ///
     /// * `base_path` - The base path where the table is located
     /// * `name` The Table name
-    /// * `params` The [OpenTableParams] to use when opening the table
+    /// * `params` The [ReadParams] to use when opening the table
     ///
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_with_params(
-        base_uri: &str,
-        name: &str,
-        params: OpenTableParams,
-    ) -> Result<Self> {
-        let path = Path::new(base_uri);
-
-        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
-            .to_str()
-            .context(InvalidTableNameSnafu { name })?;
-
-        let dataset = Dataset::open_with_params(uri, &params.open_table_params)
+    pub async fn open_with_params(uri: &str, name: &str, params: &ReadParams) -> Result<Self> {
+        let dataset = Dataset::open_with_params(uri, params)
             .await
             .map_err(|e| match e {
                 lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
@@ -103,31 +93,73 @@ impl Table {
         })
     }
 
+    /// Checkout a specific version of this [`Table`]
+    ///
+    pub async fn checkout(uri: &str, version: u64) -> Result<Self> {
+        let name = Self::get_table_name(uri)?;
+        Self::checkout_with_params(uri, &name, version, &ReadParams::default()).await
+    }
+
+    pub async fn checkout_with_name(uri: &str, name: &str, version: u64) -> Result<Self> {
+        Self::checkout_with_params(uri, name, version, &ReadParams::default()).await
+    }
+
+    pub async fn checkout_with_params(
+        uri: &str,
+        name: &str,
+        version: u64,
+        params: &ReadParams,
+    ) -> Result<Self> {
+        let dataset = Dataset::checkout_with_params(uri, version, params)
+            .await
+            .map_err(|e| match e {
+                lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
+                    name: name.to_string(),
+                },
+                e => Error::Lance {
+                    message: e.to_string(),
+                },
+            })?;
+        Ok(Table {
+            name: name.to_string(),
+            uri: uri.to_string(),
+            dataset: Arc::new(dataset),
+        })
+    }
+
+    fn get_table_name(uri: &str) -> Result<String> {
+        let path = Path::new(uri);
+        let name = path
+            .file_stem()
+            .ok_or(Error::TableNotFound {
+                name: uri.to_string(),
+            })?
+            .to_str()
+            .ok_or(Error::InvalidTableName {
+                name: uri.to_string(),
+            })?;
+        Ok(name.to_string())
+    }
+
     /// Creates a new Table
     ///
     /// # Arguments
     ///
-    /// * `base_path` - The base path where the table is located
+    /// * `uri` - The URI to the table.
     /// * `name` The Table name
-    /// * `batches` RecordBatch to be saved in the database
+    /// * `batches` RecordBatch to be saved in the database.
+    /// * `params` - Write parameters.
     ///
     /// # Returns
     ///
     /// * A [Table] object.
     pub async fn create(
-        base_uri: &str,
+        uri: &str,
         name: &str,
-        mut batches: Box<dyn RecordBatchReader>,
+        batches: impl RecordBatchReader + Send + 'static,
         params: Option<WriteParams>,
     ) -> Result<Self> {
-        let base_path = Path::new(base_uri);
-        let table_uri = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
-            .to_str()
-            .context(InvalidTableNameSnafu { name })?
-            .to_string();
-        let dataset = Dataset::write(&mut batches, &uri, params)
+        let dataset = Dataset::write(batches, uri, params)
             .await
             .map_err(|e| match e {
                 lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
@@ -139,11 +171,21 @@ impl Table {
             })?;
         Ok(Table {
             name: name.to_string(),
-            uri,
+            uri: uri.to_string(),
             dataset: Arc::new(dataset),
         })
     }
 
+    /// Schema of this Table.
+    pub fn schema(&self) -> SchemaRef {
+        Arc::new(self.dataset.schema().into())
+    }
+
+    /// Version of this Table
+    pub fn version(&self) -> u64 {
+        self.dataset.version().version
+    }
+
     /// Create index on the table.
     pub async fn create_index(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
         use lance::index::DatasetIndexExt;
@@ -176,14 +218,16 @@ impl Table {
     /// * The number of rows added
     pub async fn add(
         &mut self,
-        mut batches: Box<dyn RecordBatchReader>,
-        write_mode: Option<WriteMode>,
-    ) -> Result<usize> {
-        let mut params = WriteParams::default();
-        params.mode = write_mode.unwrap_or(WriteMode::Append);
+        batches: impl RecordBatchReader + Send + 'static,
+        params: Option<WriteParams>,
+    ) -> Result<()> {
+        let params = params.unwrap_or(WriteParams {
+            mode: WriteMode::Append,
+            ..WriteParams::default()
+        });
 
-        self.dataset = Arc::new(Dataset::write(&mut batches, &self.uri, Some(params)).await?);
-        Ok(batches.count())
+        self.dataset = Arc::new(Dataset::write(batches, &self.uri, Some(params)).await?);
+        Ok(())
     }
 
     /// Creates a new Query object that can be executed.
@@ -207,12 +251,12 @@ impl Table {
     /// Merge new data into this table.
     pub async fn merge(
         &mut self,
-        mut batches: Box<dyn RecordBatchReader>,
+        batches: impl RecordBatchReader + Send + 'static,
         left_on: &str,
         right_on: &str,
     ) -> Result<()> {
         let mut dataset = self.dataset.as_ref().clone();
-        dataset.merge(&mut batches, left_on, right_on).await?;
+        dataset.merge(batches, left_on, right_on).await?;
         self.dataset = Arc::new(dataset);
         Ok(())
     }
@@ -251,14 +295,13 @@ mod tests {
     async fn test_open() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
 
-        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        let batches = make_test_batches();
+        Dataset::write(batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
+        let table = Table::open(dataset_path.to_str().unwrap()).await.unwrap();
 
         assert_eq!(table.name, "test")
     }
@@ -267,11 +310,12 @@ mod tests {
     async fn test_open_not_found() {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
-        let table = Table::open(uri, "test").await;
+        let table = Table::open(uri).await;
         assert!(matches!(table.unwrap_err(), Error::TableNotFound { .. }));
     }
 
     #[test]
+    #[cfg(not(windows))]
     fn test_object_store_path() {
         use std::path::Path as StdPath;
         let p = StdPath::new("s3://bucket/path/to/file");
@@ -284,11 +328,11 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = make_test_batches();
+        let batches = make_test_batches();
         let _ = batches.schema().clone();
         Table::create(&uri, "test", batches, None).await.unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = make_test_batches();
+        let batches = make_test_batches();
         let result = Table::create(&uri, "test", batches, None).await;
         assert!(matches!(
             result.unwrap_err(),
@@ -301,12 +345,12 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = make_test_batches();
+        let batches = make_test_batches();
         let schema = batches.schema().clone();
         let mut table = Table::create(&uri, "test", batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+        let new_batches = RecordBatchIterator::new(
             vec![RecordBatch::try_new(
                 schema.clone(),
                 vec![Arc::new(Int32Array::from_iter_values(100..110))],
@@ -315,7 +359,7 @@ mod tests {
             .into_iter()
             .map(Ok),
             schema.clone(),
-        ));
+        );
 
         table.add(new_batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 20);
@@ -327,12 +371,12 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = tmp_dir.path().to_str().unwrap();
 
-        let batches: Box<dyn RecordBatchReader> = make_test_batches();
+        let batches = make_test_batches();
         let schema = batches.schema().clone();
         let mut table = Table::create(uri, "test", batches, None).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
-        let new_batches: Box<dyn RecordBatchReader> = Box::new(RecordBatchIterator::new(
+        let new_batches = RecordBatchIterator::new(
             vec![RecordBatch::try_new(
                 schema.clone(),
                 vec![Arc::new(Int32Array::from_iter_values(100..110))],
@@ -341,12 +385,14 @@ mod tests {
             .into_iter()
             .map(Ok),
             schema.clone(),
-        ));
+        );
 
-        table
-            .add(new_batches, Some(WriteMode::Overwrite))
-            .await
-            .unwrap();
+        let param: WriteParams = WriteParams {
+            mode: WriteMode::Overwrite,
+            ..Default::default()
+        };
+
+        table.add(new_batches, Some(param)).await.unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
         assert_eq!(table.name, "test");
     }
@@ -355,21 +401,21 @@ mod tests {
     async fn test_search() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let uri = dataset_path.to_str().unwrap();
 
-        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        let batches = make_test_batches();
+        Dataset::write(batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
+        let table = Table::open(uri).await.unwrap();
 
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = table.search(vector.clone());
         assert_eq!(vector, query.query_vector);
     }
 
-    #[derive(Default)]
+    #[derive(Default, Debug)]
     struct NoOpCacheWrapper {
         called: AtomicBool,
     }
@@ -394,10 +440,10 @@ mod tests {
     async fn test_open_table_options() {
         let tmp_dir = tempdir().unwrap();
         let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let uri = dataset_path.to_str().unwrap();
 
-        let mut batches: Box<dyn RecordBatchReader> = make_test_batches();
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        let batches = make_test_batches();
+        Dataset::write(batches, dataset_path.to_str().unwrap(), None)
             .await
             .unwrap();
 
@@ -405,27 +451,24 @@ mod tests {
 
         let mut object_store_params = ObjectStoreParams::default();
         object_store_params.object_store_wrapper = Some(wrapper.clone());
-        let param = OpenTableParams {
-            open_table_params: ReadParams {
-                store_options: Some(object_store_params),
-                ..ReadParams::default()
-            },
+        let param = ReadParams {
+            store_options: Some(object_store_params),
+            ..Default::default()
         };
-
         assert!(!wrapper.called());
-        let _ = Table::open_with_params(uri, "test", param).await.unwrap();
+        let _ = Table::open_with_params(uri, "test", &param).await.unwrap();
         assert!(wrapper.called());
     }
 
-    fn make_test_batches() -> Box<dyn RecordBatchReader> {
+    fn make_test_batches() -> impl RecordBatchReader + Send + Sync + 'static {
         let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
-        Box::new(RecordBatchIterator::new(
+        RecordBatchIterator::new(
             vec![RecordBatch::try_new(
                 schema.clone(),
                 vec![Arc::new(Int32Array::from_iter_values(0..10))],
             )],
             schema,
-        ))
+        )
     }
 
     #[tokio::test]
@@ -465,9 +508,7 @@ mod tests {
             schema,
         );
 
-        let reader: Box<dyn RecordBatchReader + Send> = Box::new(batches);
-        let mut table = Table::create(uri, "test", reader, None).await.unwrap();
-
+        let mut table = Table::create(uri, "test", batches, None).await.unwrap();
         let mut i = IvfPQIndexBuilder::new();
 
         let index_builder = i