make node available to all users

.github/workflows/make_release_commit.yml

@@ -0,0 +1,70 @@
+name: Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Just create the local commit/tags but do not push it'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Install cargo utils
+      run: cargo install cargo-bump cargo-get
+    - name: Bump vectordb
+      working-directory: rust/vectordb
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
+    - name: Bump rust/ffi/node
+      working-directory: rust/ffi/node
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV 
+    - name: Bump node
+      working-directory: node
+      run: |
+        npm version ${{ inputs.part }}
+        echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV 
+    - name: Create tag
+      run: |
+        if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and rust/ffi/node"
+          exit 1
+        fi
+        if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and node"
+          exit 1
+        fi
+        export TAG="v$CRATE_VERSION'"
+        git tag $TAG
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.RELEASE_TOKEN }}
+        branch: main
+        tags: true

.github/workflows/node.yml

@@ -67,8 +67,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: npm run test
   macos:
@@ -94,8 +96,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: |
         npm run test

.github/workflows/python.yml

@@ -30,8 +30,7 @@ jobs:
         python-version: 3.${{ matrix.python-minor-version }}
     - name: Install lancedb
       run: |
-        pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install -e ".[fts]"
         pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests
@@ -53,8 +52,7 @@ jobs:
         python-version: "3.11"
     - name: Install lancedb
       run: |
-        pip install -e .
-        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+        pip install -e ".[fts]"
         pip install pytest
     - name: Run tests
       run: pytest -x -v --durations=30 tests

.github/workflows/release.yml

@@ -0,0 +1,174 @@
+name: Prepare Release
+
+# TODO: bump versions in CI
+# NOTE: Python is a separate release for now.
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  rust:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust/vectordb
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Package Rust
+        run: cargo package --all-features
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: target/package/vectordb-*.crate
+          fail_on_unmatched_files: true
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: node/vectordb-*.tgz
+          fail_on_unmatched_files: true
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Install rustup target
+      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+      run: rustup target add aarch64-apple-darwin
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-darwin*.tgz
+        fail_on_unmatched_files: true
+
+  node-linux:
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    runs-on: ubuntu-latest
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        libc:
+          - gnu
+          # TODO: re-enable musl once we have refactored to pre-built containers
+          # Right now we have to build node from source which is too expensive.
+          # - musl
+        arch:
+          - x86_64
+          - aarch64
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Change owner to root (for npm)
+      # The docker container is run as root, so we need the files to be owned by root
+      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+      run: sudo chown -R root:root .
+    - name: Set up QEMU
+      if: ${{ matrix.arch == 'aarch64' }}
+      uses: docker/setup-qemu-action@v2  
+      with:
+        platforms: arm64
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
+      run: |
+        docker run \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
+      run: |
+        docker run --platform linux/arm64/v8 \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-linux*.tgz
+        fail_on_unmatched_files: true
+
+  release:
+    needs: [rust, node, node-macos, node-linux]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python -m twine upload --non-interactive \
+            --skip-existing \
+            --repository testpypi python/dist/*
+      - name: Publish to NPM
+        run: |
+          for filename in node/dist/*.tgz; do
+            npm publish --dry-run $filename
+          done
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
+      # - uses: softprops/action-gh-release@v1
+      #   with:
+      #     draft: false

.gitignore

@@ -4,6 +4,8 @@
 **/__pycache__
 .DS_Store
 
+.vscode
+
 rust/target
 rust/Cargo.lock
 

Cargo.lock

@@ -1052,7 +1052,6 @@ dependencies = [
  "paste",
  "petgraph",
  "rand",
- "regex",
  "uuid",
 ]
 
@@ -1646,9 +1645,9 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "0.4.17"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86dda8185bd1ffae7b910c1f68035af23be9b717c52e9cc4de176cd30b47f772"
+checksum = "fc96cf89139af6f439a0e28ccd04ddf81be795b79fda3105b7a8952fadeb778e"
 dependencies = [
  "accelerate-src",
  "arrow",
@@ -1685,7 +1684,6 @@ dependencies = [
  "rand",
  "reqwest",
  "shellexpand",
- "snafu",
  "sqlparser-lance",
  "tokio",
  "url",
@@ -3358,22 +3356,20 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
  "arrow-data",
  "arrow-schema",
  "lance",
- "object_store",
  "rand",
- "snafu",
  "tempfile",
  "tokio",
 ]
 
 [[package]]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

ci/build_linux_artifacts.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# TODO: refactor this into a Docker container we can pull
+
+set -e
+
+setup_dependencies() {
+    echo "Installing system dependencies..."
+    if [[ $1 == *musl ]]; then
+        # musllinux
+        apk add openssl-dev
+    else
+        # manylinux2014
+        yum install -y openssl-devel unzip
+    fi
+
+    if [[ $1 == x86_64* ]]; then
+        ARCH=x86_64
+    else
+        # gnu target
+        ARCH=aarch_64
+    fi
+    
+    # Install new enough protobuf (yum-provided is old)
+    PB_REL=https://github.com/protocolbuffers/protobuf/releases
+    PB_VERSION=23.1
+    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
+}
+
+install_node() {
+    echo "Installing node..."
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+    source "$HOME"/.bashrc
+
+    if [[ $1 == *musl ]]; then
+        # This node version is 15, we need 16 or higher:
+        # apk add nodejs-current npm 
+        # So instead we install from source (nvm doesn't provide binaries for musl):
+        nvm install -s --no-progress 17
+    else
+        nvm install --no-progress 17 # latest that supports glibc 2.17
+    fi
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+build_node_binary() {
+    echo "Building node library for $1..."
+    pushd node
+
+    npm ci
+    
+    if [[ $1 == *musl ]]; then
+        # This is needed for cargo to allow build cdylibs with musl
+        export RUSTFLAGS="-C target-feature=-crt-static"
+    fi
+    # We don't pass in target, since the native target here already matches
+    # and openblas-src doesn't do well with cross-compilation.
+    npm run build-release
+    npm run pack-build
+
+    popd
+}
+
+TARGET=${1:-x86_64-unknown-linux-gnu}
+# Others:
+# aarch64-unknown-linux-gnu
+# x86_64-unknown-linux-musl
+# aarch64-unknown-linux-musl
+
+setup_dependencies $TARGET
+install_node $TARGET
+install_rust
+build_node_binary $TARGET

ci/build_macos_artifacts.sh

@@ -0,0 +1,38 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/ffi/node
+
+    for target in $1
+    do
+        echo "Building rust library for $target"
+        export RUST_BACKTRACE=1
+        cargo build --release --target $target
+    done
+
+    popd
+}
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+
+prebuild_rust $targets
+build_node_binaries $targets

ci/release_process.md

@@ -0,0 +1,90 @@
+# How to release
+
+This is for the Rust crate and Node module. For now, the Python module is
+released separately.
+
+The release is started by bumping the versions and pushing a new tag. To do this
+automatically, use the `make_release_commit` GitHub action.
+
+When the tag is pushed, GitHub actions will start building the libraries and
+will upload them to a draft release.
+
+While those jobs are running, edit the release notes as needed. For example, 
+bring relevant new features and bugfixes to the top of the notes and the testing
+and CI changes to the bottom.
+
+Once the jobs have finished, the release will be marked as not draft and the
+artifacts will be released to crates.io, NPM, and PyPI.
+
+## Manual process
+
+You can also build the artifacts locally on a MacOS machine.
+
+### Build the MacOS release libraries
+
+One-time setup:
+
+```shell
+rustup target add x86_64-apple-darwin aarch64-apple-darwin
+```
+
+To build:
+
+```shell
+bash ci/build_macos_artifacts.sh
+```
+
+### Build the Linux release libraries
+
+To build a Linux library, we need to use docker with a different build script:
+
+```shell
+ARCH=aarch64
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/manylinux2014_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
+```
+
+You can change `ARCH` to `x86_64`.
+
+Similar script for musl binaries (not yet working):
+
+```shell
+ARCH=aarch64
+docker run \
+    --user $(id -u) \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
+```
+
+<!--
+
+For debugging, use these snippets:
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/manylinux2014_$ARCH \
+    bash
+```
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash
+```
+
+Note: musllinux_1_1 is Alpine Linux 3.12
+-->
+
+```
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_aarch64 \
+    bash alpine_repro.sh
+```

docs/src/fts.md

@@ -6,10 +6,9 @@ to make this available for JS as well.
 
 ## Installation
 
-To use full text search, you must install optional dependency tantivy-py:
+To use full text search, you must install the fts optional dependencies:
 
-# tantivy 0.19.2
-pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
+`pip install lancedb[fts]`
 
 
 ## Quickstart

docs/src/index.md

@@ -38,7 +38,7 @@ result = table.search([100, 100]).limit(2).to_df()
 ## Complete Demos
 
 We will be adding completed demo apps built using LanceDB.
-- [YouTube Transcript Search](../../notebooks/youtube_transcript_search.ipynb)
+- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
 
 
 ## Documentation Quick Links

node/.npmignore

@@ -0,0 +1,2 @@
+gen_test_data.py
+index.node

node/README.md

@@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
+
 ## Usage
 
 ### Basic Example
@@ -24,6 +28,19 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
+Build and install the rust library with:
+
+```bash
+npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
+```
+
+`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
+binary into an npm module called `@vectordb/<platform>` (for example, 
+`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
+module.
+
 The LanceDB javascript is built with npm:
 
 ```bash

node/examples/js-openai/index.js

@@ -1,41 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-'use strict'
-
-async function example () {
-  const lancedb = require('vectordb')
-  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
-  const apiKey = process.env.OPENAI_API_KEY
-  // The embedding function will create embeddings for the 'text' column(text in this case)
-  const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
-
-  const db = await lancedb.connect('data/sample-lancedb')
-
-  const data = [
-    { id: 1, text: 'Black T-Shirt', price: 10 },
-    { id: 2, text: 'Leather Jacket', price: 50 }
-  ]
-
-  const table = await db.createTable('vectors', data, embedding)
-  console.log(await db.tableNames())
-
-  const results = await table
-    .search('keeps me warm')
-    .limit(1)
-    .execute()
-  console.log(results[0].text)
-}
-
-example().then(_ => { console.log('All done!') })

node/examples/js-openai/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "vectordb-example-js-openai",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Lance Devs",
-  "license": "Apache-2.0",
-  "dependencies": {
-    "vectordb": "file:../..",
-    "openai": "^3.2.1"
-  }
-}

node/examples/js/package.json

@@ -9,6 +9,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/examples/ts/package.json

@@ -17,6 +17,6 @@
     "typescript": "*"
   },
   "dependencies": {
-    "vectordb": "file:../.."
+    "vectordb": "^0.1.0"
   }
 }

node/native.js

@@ -12,29 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;
 
-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`@vectordb/${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @vectordb/${currentTarget()}\`.
+
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+
+  Source error: ${e}`);
 }
 
-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;

node/package-lock.json

@@ -1,22 +1,35 @@
 {
   "name": "vectordb",
-  "version": "0.1.3",
+  "version": "0.1.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.3",
+      "version": "0.1.1",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
       "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux"
+      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
+        "@neon-rs/load": "^0.0.74",
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1",
         "apache-arrow": "^12.0.0"
       },
       "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
-        "@types/sinon": "^10.0.15",
         "@types/temp": "^0.9.1",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
@@ -27,12 +40,16 @@
         "eslint-plugin-n": "^15.7.0",
         "eslint-plugin-promise": "^6.1.1",
         "mocha": "^10.2.0",
-        "openai": "^3.2.1",
-        "sinon": "^15.1.0",
         "temp": "^0.9.4",
         "ts-node": "^10.9.1",
         "ts-node-dev": "^2.0.0",
         "typescript": "*"
+      },
+      "optionalDependencies": {
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -200,6 +217,20 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -235,50 +266,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/@sinonjs/commons": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
-      "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
-      "dev": true,
-      "dependencies": {
-        "type-detect": "4.0.8"
-      }
-    },
-    "node_modules/@sinonjs/fake-timers": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz",
-      "integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==",
-      "dev": true,
-      "dependencies": {
-        "@sinonjs/commons": "^3.0.0"
-      }
-    },
-    "node_modules/@sinonjs/samsam": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz",
-      "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==",
-      "dev": true,
-      "dependencies": {
-        "@sinonjs/commons": "^2.0.0",
-        "lodash.get": "^4.4.2",
-        "type-detect": "^4.0.8"
-      }
-    },
-    "node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
-      "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
-      "dev": true,
-      "dependencies": {
-        "type-detect": "4.0.8"
-      }
-    },
-    "node_modules/@sinonjs/text-encoding": {
-      "version": "0.7.2",
-      "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz",
-      "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
-      "dev": true
-    },
     "node_modules/@tsconfig/node10": {
       "version": "1.0.9",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
@@ -354,21 +341,6 @@
       "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
       "dev": true
     },
-    "node_modules/@types/sinon": {
-      "version": "10.0.15",
-      "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.15.tgz",
-      "integrity": "sha512-3lrFNQG0Kr2LDzvjyjB6AMJk4ge+8iYhQfdnSwIwlG88FUOV43kPcQqDZkDa/h3WSZy6i8Fr0BSjfQtB1B3xuQ==",
-      "dev": true,
-      "dependencies": {
-        "@types/sinonjs__fake-timers": "*"
-      }
-    },
-    "node_modules/@types/sinonjs__fake-timers": {
-      "version": "8.1.2",
-      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz",
-      "integrity": "sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==",
-      "dev": true
-    },
     "node_modules/@types/strip-bom": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -806,12 +778,6 @@
         "node": "*"
       }
     },
-    "node_modules/asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
-    },
     "node_modules/available-typed-arrays": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
@@ -824,15 +790,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
-      "dependencies": {
-        "follow-redirects": "^1.14.8"
-      }
-    },
     "node_modules/balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -1045,18 +1002,6 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "node_modules/combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
-      "dependencies": {
-        "delayed-stream": "~1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.8"
-      }
-    },
     "node_modules/command-line-args": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
@@ -1268,15 +1213,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.4.0"
-      }
-    },
     "node_modules/diff": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -2040,26 +1976,6 @@
       "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
       "dev": true
     },
-    "node_modules/follow-redirects": {
-      "version": "1.15.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true,
-      "funding": [
-        {
-          "type": "individual",
-          "url": "https://github.com/sponsors/RubenVerborgh"
-        }
-      ],
-      "engines": {
-        "node": ">=4.0"
-      },
-      "peerDependenciesMeta": {
-        "debug": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/for-each": {
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
@@ -2069,20 +1985,6 @@
         "is-callable": "^1.1.3"
       }
     },
-    "node_modules/form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
-      "dependencies": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -2716,12 +2618,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/isarray": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-      "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==",
-      "dev": true
-    },
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -2782,12 +2678,6 @@
         "json5": "lib/cli.js"
       }
     },
-    "node_modules/just-extend": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
-      "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
-      "dev": true
-    },
     "node_modules/levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -2821,12 +2711,6 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
-    "node_modules/lodash.get": {
-      "version": "4.4.2",
-      "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
-      "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==",
-      "dev": true
-    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -2898,27 +2782,6 @@
         "node": ">=8.6"
       }
     },
-    "node_modules/mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true,
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
-    "node_modules/mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
-      "dependencies": {
-        "mime-db": "1.52.0"
-      },
-      "engines": {
-        "node": ">= 0.6"
-      }
-    },
     "node_modules/minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -3096,28 +2959,6 @@
       "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==",
       "dev": true
     },
-    "node_modules/nise": {
-      "version": "5.1.4",
-      "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.4.tgz",
-      "integrity": "sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==",
-      "dev": true,
-      "dependencies": {
-        "@sinonjs/commons": "^2.0.0",
-        "@sinonjs/fake-timers": "^10.0.2",
-        "@sinonjs/text-encoding": "^0.7.1",
-        "just-extend": "^4.0.2",
-        "path-to-regexp": "^1.7.0"
-      }
-    },
-    "node_modules/nise/node_modules/@sinonjs/commons": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
-      "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
-      "dev": true,
-      "dependencies": {
-        "type-detect": "4.0.8"
-      }
-    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -3189,16 +3030,6 @@
         "wrappy": "1"
       }
     },
-    "node_modules/openai": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz",
-      "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==",
-      "dev": true,
-      "dependencies": {
-        "axios": "^0.26.0",
-        "form-data": "^4.0.0"
-      }
-    },
     "node_modules/optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -3302,15 +3133,6 @@
       "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
-    "node_modules/path-to-regexp": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
-      "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
-      "dev": true,
-      "dependencies": {
-        "isarray": "0.0.1"
-      }
-    },
     "node_modules/path-type": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
@@ -3618,45 +3440,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/sinon": {
-      "version": "15.1.0",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz",
-      "integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==",
-      "dev": true,
-      "dependencies": {
-        "@sinonjs/commons": "^3.0.0",
-        "@sinonjs/fake-timers": "^10.2.0",
-        "@sinonjs/samsam": "^8.0.0",
-        "diff": "^5.1.0",
-        "nise": "^5.1.4",
-        "supports-color": "^7.2.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/sinon"
-      }
-    },
-    "node_modules/sinon/node_modules/diff": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
-      "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.3.1"
-      }
-    },
-    "node_modules/sinon/node_modules/supports-color": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dev": true,
-      "dependencies": {
-        "has-flag": "^4.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
@@ -4442,6 +4225,17 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4468,52 +4262,6 @@
         "fastq": "^1.6.0"
       }
     },
-    "@sinonjs/commons": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
-      "integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
-      "dev": true,
-      "requires": {
-        "type-detect": "4.0.8"
-      }
-    },
-    "@sinonjs/fake-timers": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz",
-      "integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==",
-      "dev": true,
-      "requires": {
-        "@sinonjs/commons": "^3.0.0"
-      }
-    },
-    "@sinonjs/samsam": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz",
-      "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==",
-      "dev": true,
-      "requires": {
-        "@sinonjs/commons": "^2.0.0",
-        "lodash.get": "^4.4.2",
-        "type-detect": "^4.0.8"
-      },
-      "dependencies": {
-        "@sinonjs/commons": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
-          "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
-          "dev": true,
-          "requires": {
-            "type-detect": "4.0.8"
-          }
-        }
-      }
-    },
-    "@sinonjs/text-encoding": {
-      "version": "0.7.2",
-      "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz",
-      "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
-      "dev": true
-    },
     "@tsconfig/node10": {
       "version": "1.0.9",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
@@ -4589,21 +4337,6 @@
       "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
       "dev": true
     },
-    "@types/sinon": {
-      "version": "10.0.15",
-      "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.15.tgz",
-      "integrity": "sha512-3lrFNQG0Kr2LDzvjyjB6AMJk4ge+8iYhQfdnSwIwlG88FUOV43kPcQqDZkDa/h3WSZy6i8Fr0BSjfQtB1B3xuQ==",
-      "dev": true,
-      "requires": {
-        "@types/sinonjs__fake-timers": "*"
-      }
-    },
-    "@types/sinonjs__fake-timers": {
-      "version": "8.1.2",
-      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz",
-      "integrity": "sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==",
-      "dev": true
-    },
     "@types/strip-bom": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -4891,27 +4624,12 @@
       "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
       "dev": true
     },
-    "asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
-    },
     "available-typed-arrays": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
       "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
       "dev": true
     },
-    "axios": {
-      "version": "0.26.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
-      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
-      "dev": true,
-      "requires": {
-        "follow-redirects": "^1.14.8"
-      }
-    },
     "balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -5076,15 +4794,6 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
       "dev": true
     },
-    "combined-stream": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
-      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
-      "requires": {
-        "delayed-stream": "~1.0.0"
-      }
-    },
     "command-line-args": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
@@ -5244,12 +4953,6 @@
         "object-keys": "^1.1.1"
       }
     },
-    "delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true
-    },
     "diff": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -5817,12 +5520,6 @@
       "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
       "dev": true
     },
-    "follow-redirects": {
-      "version": "1.15.2",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
-      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
-      "dev": true
-    },
     "for-each": {
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
@@ -5832,17 +5529,6 @@
         "is-callable": "^1.1.3"
       }
     },
-    "form-data": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
-      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
-      "requires": {
-        "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.8",
-        "mime-types": "^2.1.12"
-      }
-    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -6271,12 +5957,6 @@
         "call-bind": "^1.0.2"
       }
     },
-    "isarray": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-      "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==",
-      "dev": true
-    },
     "isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -6324,12 +6004,6 @@
         "minimist": "^1.2.0"
       }
     },
-    "just-extend": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
-      "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
-      "dev": true
-    },
     "levn": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -6354,12 +6028,6 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
-    "lodash.get": {
-      "version": "4.4.2",
-      "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
-      "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==",
-      "dev": true
-    },
     "lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -6416,21 +6084,6 @@
         "picomatch": "^2.3.1"
       }
     },
-    "mime-db": {
-      "version": "1.52.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
-      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
-      "dev": true
-    },
-    "mime-types": {
-      "version": "2.1.35",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
-      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
-      "dev": true,
-      "requires": {
-        "mime-db": "1.52.0"
-      }
-    },
     "minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -6564,30 +6217,6 @@
       "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==",
       "dev": true
     },
-    "nise": {
-      "version": "5.1.4",
-      "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.4.tgz",
-      "integrity": "sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==",
-      "dev": true,
-      "requires": {
-        "@sinonjs/commons": "^2.0.0",
-        "@sinonjs/fake-timers": "^10.0.2",
-        "@sinonjs/text-encoding": "^0.7.1",
-        "just-extend": "^4.0.2",
-        "path-to-regexp": "^1.7.0"
-      },
-      "dependencies": {
-        "@sinonjs/commons": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
-          "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
-          "dev": true,
-          "requires": {
-            "type-detect": "4.0.8"
-          }
-        }
-      }
-    },
     "normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -6638,16 +6267,6 @@
         "wrappy": "1"
       }
     },
-    "openai": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz",
-      "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==",
-      "dev": true,
-      "requires": {
-        "axios": "^0.26.0",
-        "form-data": "^4.0.0"
-      }
-    },
     "optionator": {
       "version": "0.9.1",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
@@ -6721,15 +6340,6 @@
       "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
       "dev": true
     },
-    "path-to-regexp": {
-      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
-      "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
-      "dev": true,
-      "requires": {
-        "isarray": "0.0.1"
-      }
-    },
     "path-type": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
@@ -6919,37 +6529,6 @@
         "object-inspect": "^1.9.0"
       }
     },
-    "sinon": {
-      "version": "15.1.0",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz",
-      "integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==",
-      "dev": true,
-      "requires": {
-        "@sinonjs/commons": "^3.0.0",
-        "@sinonjs/fake-timers": "^10.2.0",
-        "@sinonjs/samsam": "^8.0.0",
-        "diff": "^5.1.0",
-        "nise": "^5.1.4",
-        "supports-color": "^7.2.0"
-      },
-      "dependencies": {
-        "diff": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
-          "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==",
-          "dev": true
-        },
-        "supports-color": {
-          "version": "7.2.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-          "dev": true,
-          "requires": {
-            "has-flag": "^4.0.0"
-          }
-        }
-      }
-    },
     "slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",

node/package.json

@@ -1,15 +1,18 @@
 {
   "name": "vectordb",
-  "version": "0.1.3",
+  "version": "0.1.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts"
+    "lint": "eslint src --ext .js,.ts",
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
   },
   "repository": {
     "type": "git",
@@ -24,10 +27,10 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
-    "@types/sinon": "^10.0.15",
     "@types/temp": "^0.9.1",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
     "cargo-cp-artifact": "^0.1",
@@ -38,8 +41,6 @@
     "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "mocha": "^10.2.0",
-    "sinon": "^15.1.0",
-    "openai": "^3.2.1",
     "temp": "^0.9.4",
     "ts-node": "^10.9.1",
     "ts-node-dev": "^2.0.0",
@@ -47,6 +48,33 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
     "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
+      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
+      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
+    }
+  },
+  "optionalDependencies": {
+    "@vectordb/darwin-arm64": "0.1.2",
+    "@vectordb/darwin-x64": "0.1.2",
+    "@vectordb/linux-x64-gnu": "0.1.2",
+    "@vectordb/linux-x64-musl": "0.1.2",
+    "@vectordb/linux-arm64-gnu": "0.1.2",
+    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

node/src/arrow.ts

@@ -15,16 +15,15 @@
 import {
   Field,
   Float32,
-  List, type ListBuilder,
+  List,
   makeBuilder,
   RecordBatchFileWriter,
   Table, Utf8,
   type Vector,
   vectorFromArray
 } from 'apache-arrow'
-import { type EmbeddingFunction } from './index'
 
-export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
+export function convertToTable (data: Array<Record<string, unknown>>): Table {
   if (data.length === 0) {
     throw new Error('At least one record needs to be provided')
   }
@@ -34,7 +33,11 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
 
   for (const columnsKey of columns) {
     if (columnsKey === 'vector') {
-      const listBuilder = newVectorListBuilder()
+      const children = new Field<Float32>('item', new Float32())
+      const list = new List(children)
+      const listBuilder = makeBuilder({
+        type: list
+      })
       const vectorSize = (data[0].vector as any[]).length
       for (const datum of data) {
         if ((datum[columnsKey] as any[]).length !== vectorSize) {
@@ -49,14 +52,6 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
       for (const datum of data) {
         values.push(datum[columnsKey])
       }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        const listBuilder = newVectorListBuilder()
-        vectors.map(v => listBuilder.append(v))
-        records.vector = listBuilder.finish().toVector()
-      }
-
       if (typeof values[0] === 'string') {
         // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
         records[columnsKey] = vectorFromArray(values, new Utf8())
@@ -69,17 +64,8 @@ export async function convertToTable<T> (data: Array<Record<string, unknown>>, e
   return new Table(records)
 }
 
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorListBuilder (): ListBuilder<Float32, any> {
-  const children = new Field<Float32>('item', new Float32())
-  const list = new List(children)
-  return makeBuilder({
-    type: list
-  })
-}
-
-export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
-  const table = await convertToTable(data, embeddings)
+export async function fromRecordsToBuffer (data: Array<Record<string, unknown>>): Promise<Buffer> {
+  const table = convertToTable(data)
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }

node/src/embedding/embedding_function.ts

@@ -1,28 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export interface EmbeddingFunction<T> {
-  /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
-  sourceColumn: string
-
-  /**
-     * Creates a vector representation for the given values.
-     */
-  embed: (data: T[]) => Promise<number[][]>
-}

node/src/embedding/openai.ts

@@ -1,51 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { type EmbeddingFunction } from '../index'
-
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: any
-  private readonly _modelName: string
-
-  constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
-    let openai
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      openai = require('openai')
-    } catch {
-      throw new Error('please install openai using npm install openai')
-    }
-
-    this.sourceColumn = sourceColumn
-    const configuration = new openai.Configuration({
-      apiKey: openAIKey
-    })
-    this._openai = new openai.OpenAIApi(configuration)
-    this._modelName = modelName
-  }
-
-  async embed (data: string[]): Promise<number[][]> {
-    const response = await this._openai.createEmbedding({
-      model: this._modelName,
-      input: data
-    })
-    const embeddings: number[][] = []
-    for (let i = 0; i < response.data.data.length; i++) {
-      embeddings.push(response.data.data[i].embedding as number[])
-    }
-    return embeddings
-  }
-
-  sourceColumn: string
-}

node/src/index.ts

@@ -19,21 +19,16 @@ import {
   Vector
 } from 'apache-arrow'
 import { fromRecordsToBuffer } from './arrow'
-import type { EmbeddingFunction } from './embedding/embedding_function'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex } = require('../native.js')
 
-export type { EmbeddingFunction }
-export { OpenAIEmbeddingFunction } from './embedding/openai'
-
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
  */
 export async function connect (uri: string): Promise<Connection> {
-  const db = await databaseNew(uri)
-  return new Connection(db, uri)
+  return new Connection(uri)
 }
 
 /**
@@ -43,9 +38,9 @@ export class Connection {
   private readonly _uri: string
   private readonly _db: any
 
-  constructor (db: any, uri: string) {
+  constructor (uri: string) {
     this._uri = uri
-    this._db = db
+    this._db = databaseNew(uri)
   }
 
   get uri (): string {
@@ -60,50 +55,17 @@ export class Connection {
   }
 
   /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   */
-  async openTable (name: string): Promise<Table>
-  /**
-   * Open a table in the database.
-   *
-   * @param name The name of the table.
-   * @param embeddings An embedding function to use on this Table
-   */
-  async openTable<T> (name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async openTable<T> (name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
+     * Open a table in the database.
+     * @param name The name of the table.
+     */
+  async openTable (name: string): Promise<Table> {
     const tbl = await databaseOpenTable.call(this._db, name)
-    if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
-    } else {
-      return new Table(tbl, name)
-    }
+    return new Table(tbl, name)
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   */
-
-  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-  /**
-   * Creates a new Table and initialize it with new data.
-   *
-   * @param name The name of the table.
-   * @param data Non-empty Array of Records to be inserted into the Table
-   * @param embeddings An embedding function to use on this Table
-   */
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  async createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table<T>> {
-    const tbl = await tableCreate.call(this._db, name, await fromRecordsToBuffer(data, embeddings))
-    if (embeddings !== undefined) {
-      return new Table(tbl, name, embeddings)
-    } else {
-      return new Table(tbl, name)
-    }
+  async createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table> {
+    await tableCreate.call(this._db, name, await fromRecordsToBuffer(data))
+    return await this.openTable(name)
   }
 
   async createTableArrow (name: string, table: ArrowTable): Promise<Table> {
@@ -113,22 +75,16 @@ export class Connection {
   }
 }
 
-export class Table<T = number[]> {
+/**
+ * A table in a LanceDB database.
+ */
+export class Table {
   private readonly _tbl: any
   private readonly _name: string
-  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, name: string)
-  /**
-   * @param tbl
-   * @param name
-   * @param embeddings An embedding function to use when interacting with this table
-   */
-  constructor (tbl: any, name: string, embeddings: EmbeddingFunction<T>)
-  constructor (tbl: any, name: string, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, name: string) {
     this._tbl = tbl
     this._name = name
-    this._embeddings = embeddings
   }
 
   get name (): string {
@@ -136,11 +92,11 @@ export class Table<T = number[]> {
   }
 
   /**
-   * Creates a search query to find the nearest neighbors of the given search term
-   * @param query The query search term
-   */
-  search (query: T): Query<T> {
-    return new Query(this._tbl, query, this._embeddings)
+     * Create a search query to find the nearest neighbors of the given query vector.
+     * @param queryVector The query vector.
+     */
+  search (queryVector: number[]): Query {
+    return new Query(this._tbl, queryVector)
   }
 
   /**
@@ -150,7 +106,7 @@ export class Table<T = number[]> {
    * @return The number of rows added to the table
    */
   async add (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Append.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Append.toString())
   }
 
   /**
@@ -160,14 +116,9 @@ export class Table<T = number[]> {
    * @return The number of rows added to the table
    */
   async overwrite (data: Array<Record<string, unknown>>): Promise<number> {
-    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data, this._embeddings), WriteMode.Overwrite.toString())
+    return tableAdd.call(this._tbl, await fromRecordsToBuffer(data), WriteMode.Overwrite.toString())
   }
 
-  /**
-   * Create an ANN index on this Table vector index.
-   *
-   * @param indexParams The parameters of this Index, @see VectorIndexParams.
-   */
   async create_index (indexParams: VectorIndexParams): Promise<any> {
     return tableCreateVectorIndex.call(this._tbl, indexParams)
   }
@@ -226,35 +177,32 @@ export type VectorIndexParams = IvfPQIndexConfig
 /**
  * A builder for nearest neighbor queries for LanceDB.
  */
-export class Query<T = number[]> {
+export class Query {
   private readonly _tbl: any
-  private readonly _query: T
-  private _queryVector?: number[]
+  private readonly _queryVector: number[]
   private _limit: number
   private _refineFactor?: number
   private _nprobes: number
   private readonly _columns?: string[]
   private _filter?: string
   private _metricType?: MetricType
-  private readonly _embeddings?: EmbeddingFunction<T>
 
-  constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
+  constructor (tbl: any, queryVector: number[]) {
     this._tbl = tbl
-    this._query = query
+    this._queryVector = queryVector
     this._limit = 10
     this._nprobes = 20
     this._refineFactor = undefined
     this._columns = undefined
     this._filter = undefined
     this._metricType = undefined
-    this._embeddings = embeddings
   }
 
   /***
    * Sets the number of results that will be returned
    * @param value number of results
    */
-  limit (value: number): Query<T> {
+  limit (value: number): Query {
     this._limit = value
     return this
   }
@@ -263,7 +211,7 @@ export class Query<T = number[]> {
    * Refine the results by reading extra elements and re-ranking them in memory.
    * @param value refine factor to use in this query.
    */
-  refineFactor (value: number): Query<T> {
+  refineFactor (value: number): Query {
     this._refineFactor = value
     return this
   }
@@ -272,7 +220,7 @@ export class Query<T = number[]> {
    * The number of probes used. A higher number makes search more accurate but also slower.
    * @param value The number of probes used.
    */
-  nprobes (value: number): Query<T> {
+  nprobes (value: number): Query {
     this._nprobes = value
     return this
   }
@@ -281,7 +229,7 @@ export class Query<T = number[]> {
    * A filter statement to be applied to this query.
    * @param value A filter in the same format used by a sql WHERE clause.
    */
-  filter (value: string): Query<T> {
+  filter (value: string): Query {
     this._filter = value
     return this
   }
@@ -290,7 +238,7 @@ export class Query<T = number[]> {
    * The MetricType used for this Query.
    * @param value The metric to the. @see MetricType for the different options
    */
-  metricType (value: MetricType): Query<T> {
+  metricType (value: MetricType): Query {
     this._metricType = value
     return this
   }
@@ -299,12 +247,6 @@ export class Query<T = number[]> {
    * Execute the query and return the results as an Array of Objects
    */
   async execute<T = Record<string, unknown>> (): Promise<T[]> {
-    if (this._embeddings !== undefined) {
-      this._queryVector = (await this._embeddings.embed([this._query]))[0]
-    } else {
-      this._queryVector = this._query as number[]
-    }
-
     const buffer = await tableSearch.call(this._tbl, this)
     const data = tableFromIPC(buffer)
     return data.toArray().map((entry: Record<string, unknown>) => {

node/src/test/embedding/openai.ts

@@ -1,50 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import { OpenAIEmbeddingFunction } from '../../embedding/openai'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { OpenAIApi } = require('openai')
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const { stub } = require('sinon')
-
-describe('OpenAPIEmbeddings', function () {
-  const stubValue = {
-    data: {
-      data: [
-        {
-          embedding: Array(1536).fill(1.0)
-        },
-        {
-          embedding: Array(1536).fill(2.0)
-        }
-      ]
-    }
-  }
-
-  describe('#embed', function () {
-    it('should create vector embeddings', async function () {
-      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
-      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
-      const vectors = await f.embed(['abc', 'def'])
-      assert.isTrue(openAIStub.calledOnce)
-      assert.equal(vectors.length, 2)
-      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
-      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
-    })
-  })
-})

node/src/test/io.ts

@@ -1,52 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// IO tests
-
-import { describe } from 'mocha'
-import { assert } from 'chai'
-
-import * as lancedb from '../index'
-
-describe('LanceDB S3 client', function () {
-  if (process.env.TEST_S3_BASE_URL != null) {
-    const baseUri = process.env.TEST_S3_BASE_URL
-    it('should have a valid url', async function () {
-      const uri = `${baseUri}/valid_url`
-      const table = await createTestDB(uri, 2, 20)
-      const con = await lancedb.connect(uri)
-      assert.equal(con.uri, uri)
-
-      const results = await table.search([0.1, 0.3]).limit(5).execute()
-      assert.equal(results.length, 5)
-    })
-  } else {
-    describe.skip('Skip S3 test', function () {})
-  }
-})
-
-async function createTestDB (uri: string, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
-  const con = await lancedb.connect(uri)
-
-  const data = []
-  for (let i = 0; i < numRows; i++) {
-    const vector = []
-    for (let j = 0; j < numDimensions; j++) {
-      vector.push(i + (j * 0.1))
-    }
-    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
-  }
-
-  return await con.createTable('vectors', data)
-}

node/src/test/test.ts

@@ -17,7 +17,7 @@ import { assert } from 'chai'
 import { track } from 'temp'
 
 import * as lancedb from '../index'
-import { type EmbeddingFunction, MetricType, Query } from '../index'
+import { MetricType, Query } from '../index'
 
 describe('LanceDB client', function () {
   describe('when creating a connection to lancedb', function () {
@@ -140,39 +140,6 @@ describe('LanceDB client', function () {
       await table.create_index({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 })
     }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
   })
-
-  describe('when using a custom embedding function', function () {
-    class TextEmbedding implements EmbeddingFunction<string> {
-      sourceColumn: string
-
-      constructor (targetColumn: string) {
-        this.sourceColumn = targetColumn
-      }
-
-      _embedding_map = new Map<string, number[]>([
-        ['foo', [2.1, 2.2]],
-        ['bar', [3.1, 3.2]]
-      ])
-
-      async embed (data: string[]): Promise<number[][]> {
-        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
-      }
-    }
-
-    it('should encode the original data into embeddings', async function () {
-      const dir = await track().mkdir('lancejs')
-      const con = await lancedb.connect(dir)
-      const embeddings = new TextEmbedding('name')
-
-      const data = [
-        { price: 10, name: 'foo' },
-        { price: 50, name: 'bar' }
-      ]
-      const table = await con.createTable('vectors', data, embeddings)
-      const results = await table.search('foo').execute()
-      assert.equal(results.length, 2)
-    })
-  })
 })
 
 describe('Query object', function () {

notebooks/diffusiondb/datagen.py

@@ -1,108 +0,0 @@
-#!/usr/bin/env python
-#
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""Dataset hf://poloclub/diffusiondb
-"""
-
-import io
-from argparse import ArgumentParser
-from multiprocessing import Pool
-
-import lance
-import lancedb
-import pyarrow as pa
-from datasets import load_dataset
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
-
-MODEL_ID = "openai/clip-vit-base-patch32"
-
-device = "cuda"
-
-tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
-model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
-processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-
-schema = pa.schema(
-    [
-        pa.field("prompt", pa.string()),
-        pa.field("seed", pa.uint32()),
-        pa.field("step", pa.uint16()),
-        pa.field("cfg", pa.float32()),
-        pa.field("sampler", pa.string()),
-        pa.field("width", pa.uint16()),
-        pa.field("height", pa.uint16()),
-        pa.field("timestamp", pa.timestamp("s")),
-        pa.field("image_nsfw", pa.float32()),
-        pa.field("prompt_nsfw", pa.float32()),
-        pa.field("vector", pa.list_(pa.float32(), 512)),
-        pa.field("image", pa.binary()),
-    ]
-)
-
-
-def pil_to_bytes(img) -> list[bytes]:
-    buf = io.BytesIO()
-    img.save(buf, format="PNG")
-    return buf.getvalue()
-
-
-def generate_clip_embeddings(batch) -> pa.RecordBatch:
-    image = processor(text=None, images=batch["image"], return_tensors="pt")[
-        "pixel_values"
-    ].to(device)
-    img_emb = model.get_image_features(image)
-    batch["vector"] = img_emb.cpu().tolist()
-
-    with Pool() as p:
-        batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
-    return batch
-
-
-def datagen(args):
-    """Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
-    dataset = load_dataset("poloclub/diffusiondb", args.subset)
-    data = []
-    for b in dataset.map(
-        generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
-    )["train"]:
-        b["image"] = b["image_bytes"]
-        del b["image_bytes"]
-        data.append(b)
-    tbl = pa.Table.from_pylist(data, schema=schema)
-    return tbl
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        "-o", "--output", metavar="DIR", help="Output lance directory", required=True
-    )
-    parser.add_argument(
-        "-s",
-        "--subset",
-        choices=["2m_all", "2m_first_10k", "2m_first_100k"],
-        default="2m_first_10k",
-        help="subset of the hg dataset",
-    )
-
-    args = parser.parse_args()
-
-    batches = datagen(args)
-    lance.write_dataset(batches, args.output)
-
-
-if __name__ == "__main__":
-    main()

notebooks/diffusiondb/requirements.txt

@@ -1,9 +0,0 @@
-datasets
-Pillow
-lancedb
-isort
-black
-transformers
---index-url https://download.pytorch.org/whl/cu118
-torch
-torchvision

notebooks/multimodal_search.ipynb

@@ -1,240 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install --quiet -U lancedb\n",
-    "!pip install --quiet gradio transformers torch torchvision"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 60,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import io\n",
-    "import PIL\n",
-    "import duckdb\n",
-    "import lancedb"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## First run setup: Download data and pre-process"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<lance.dataset.LanceDataset at 0x3045db590>"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# remove null prompts\n",
-    "import lance\n",
-    "import pyarrow.compute as pc\n",
-    "\n",
-    "# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
-    "data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
-    "\n",
-    "# First data processing and full-text-search index\n",
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
-    "tbl = tbl.create_fts_index([\"prompt\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create / Open LanceDB Table"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "db = lancedb.connect(\"~/datasets/demo\")\n",
-    "tbl = db.open_table(\"diffusiondb\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create CLIP embedding function for the text"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 63,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
-    "\n",
-    "MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
-    "\n",
-    "tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
-    "model = CLIPModel.from_pretrained(MODEL_ID)\n",
-    "processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
-    "\n",
-    "def embed_func(query):\n",
-    "    inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
-    "    text_features = model.get_text_features(**inputs)\n",
-    "    return text_features.detach().numpy()[0]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search functions for Gradio"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 64,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def find_image_vectors(query):\n",
-    "    emb = embed_func(query)\n",
-    "    return _extract(tbl.search(emb).limit(9).to_df())\n",
-    "\n",
-    "def find_image_keywords(query):\n",
-    "    return _extract(tbl.search(query).limit(9).to_df())\n",
-    "\n",
-    "def find_image_sql(query):\n",
-    "    diffusiondb = tbl.to_lance()\n",
-    "    return _extract(duckdb.query(query).to_df())\n",
-    "\n",
-    "def _extract(df):\n",
-    "    image_col = \"image\"\n",
-    "    return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup Gradio interface"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 65,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Running on local URL:  http://127.0.0.1:7867\n",
-      "\n",
-      "To create a public link, set `share=True` in `launch()`.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/html": [
-       "<div><iframe src=\"http://127.0.0.1:7867/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
-      ],
-      "text/plain": [
-       "<IPython.core.display.HTML object>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": []
-     },
-     "execution_count": 65,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import gradio as gr\n",
-    "\n",
-    "\n",
-    "with gr.Blocks() as demo:\n",
-    "\n",
-    "    with gr.Row():\n",
-    "        with gr.Tab(\"Embeddings\"):\n",
-    "            vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
-    "            b1 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"Keywords\"):\n",
-    "            keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
-    "            b2 = gr.Button(\"Submit\")\n",
-    "        with gr.Tab(\"SQL\"):\n",
-    "            sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
-    "            b3 = gr.Button(\"Submit\")\n",
-    "    with gr.Row():\n",
-    "        gallery = gr.Gallery(\n",
-    "                label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
-    "            ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\")   \n",
-    "        \n",
-    "    b1.click(find_image_vectors, inputs=vector_query, outputs=gallery)\n",
-    "    b2.click(find_image_keywords, inputs=keyword_query, outputs=gallery)\n",
-    "    b3.click(find_image_sql, inputs=sql_query, outputs=gallery)\n",
-    "    \n",
-    "demo.launch()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}

python/lancedb/db.py

@@ -13,16 +13,13 @@
 
 from __future__ import annotations
 
-import os
 from pathlib import Path
-import os
 
 import pyarrow as pa
-from pyarrow import fs
 
 from .common import DATA, URI
 from .table import LanceTable
-from .util import get_uri_scheme, get_uri_location
+from .util import get_uri_scheme
 
 
 class LanceDBConnection:
@@ -50,20 +47,11 @@ class LanceDBConnection:
         -------
         A list of table names.
         """
-        try:
-            filesystem, path = fs.FileSystem.from_uri(self.uri)
-        except pa.ArrowInvalid:
-            raise NotImplementedError(
-                "Unsupported scheme: " + self.uri
-            )
-
-        try:
-            paths = filesystem.get_file_info(fs.FileSelector(get_uri_location(self.uri)))
-        except FileNotFoundError:
-            # It is ok if the file does not exist since it will be created
-            paths = []
-        tables = [os.path.splitext(file_info.base_name)[0] for file_info in paths if file_info.extension == 'lance']
-        return tables
+        if get_uri_scheme(self.uri) == "file":
+            return [p.stem for p in Path(self.uri).glob("*.lance")]
+        raise NotImplementedError(
+            "List table_names is only supported for local filesystem for now"
+        )
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -124,15 +112,3 @@ class LanceDBConnection:
         A LanceTable object representing the table.
         """
         return LanceTable(self, name)
-
-    def drop_table(self, name: str):
-        """Drop a table from the database.
-
-        Parameters
-        ----------
-        name: str
-            The name of the table.
-        """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
-        table_path = os.path.join(path, name + ".lance")
-        filesystem.delete_dir(table_path)

python/lancedb/fts.py

@@ -16,13 +16,7 @@ import os
 from typing import List, Tuple
 
 import pyarrow as pa
-
-try:
-    import tantivy
-except ImportError:
-    raise ImportError(
-        "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
-    )
+import tantivy
 
 from .table import LanceTable
 

python/lancedb/query.py

@@ -153,7 +153,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             import tantivy
         except ImportError:
             raise ImportError(
-                "Please install tantivy-py `pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985` to use the full text search feature."
+                "You need to install the `lancedb[fts]` extra to use this method."
             )
 
         from .fts import search_index

python/lancedb/table.py

@@ -253,7 +253,8 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
     vector_column_name: str
         The name of the vector column.
     """
-    if vector_column_name not in data.column_names:
+    i = data.column_names.index(vector_column_name)
+    if i < 0:
         raise ValueError(f"Missing vector column: {vector_column_name}")
     vec_arr = data[vector_column_name].combine_chunks()
     if pa.types.is_fixed_size_list(vec_arr.type):
@@ -265,4 +266,4 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
         values = values.cast(pa.float32())
     list_size = len(values) / len(data)
     vec_arr = pa.FixedSizeListArray.from_arrays(values, list_size)
-    return data.set_column(data.column_names.index(vector_column_name), vector_column_name, vec_arr)
+    return data.set_column(i, vector_column_name, vec_arr)

python/lancedb/util.py

@@ -41,23 +41,3 @@ def get_uri_scheme(uri: str) -> str:
         # So we add special handling here for schemes that are a single character
         scheme = "file"
     return scheme
-
-
-def get_uri_location(uri: str) -> str:
-    """
-    Get the location of a URI. If the parameter is not a url, assumes it is just a path
-
-    Parameters
-    ----------
-    uri : str
-        The URI to parse.
-
-    Returns
-    -------
-    str: Location part of the URL, without scheme
-    """
-    parsed = urlparse(uri)
-    if not parsed.netloc:
-        return parsed.path
-    else:
-        return parsed.netloc + parsed.path

python/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "lancedb"
-version = "0.1.5"
-dependencies = ["pylance>=0.4.17", "ratelimiter", "retry", "tqdm"]
+version = "0.1.2"
+dependencies = ["pylance>=0.4.6", "ratelimiter", "retry", "tqdm"]
 description = "lancedb"
 authors = [
     { name = "LanceDB Devs", email = "dev@lancedb.com" },
@@ -45,6 +45,10 @@ dev = [
 docs = [
     "mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"
 ]
+fts = [
+    # tantivy 0.19.2
+    "tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985"
+]
 
 [build-system]
 requires = [

python/tests/test_db.py

@@ -97,26 +97,3 @@ def test_create_mode(tmp_path):
     )
     tbl = db.create_table("test", data=new_data, mode="overwrite")
     assert tbl.to_pandas().item.tolist() == ["fizz", "buzz"]
-
-
-def test_delete_table(tmp_path):
-    db = lancedb.connect(tmp_path)
-    data = pd.DataFrame(
-        {
-            "vector": [[3.1, 4.1], [5.9, 26.5]],
-            "item": ["foo", "bar"],
-            "price": [10.0, 20.0],
-        }
-    )
-    db.create_table("test", data=data)
-
-    with pytest.raises(Exception):
-        db.create_table("test", data=data)
-
-    assert db.table_names() == ["test"]
-
-    db.drop_table("test")
-    assert db.table_names() == []
-
-    db.create_table("test", data=data)
-    assert db.table_names() == ["test"]

python/tests/test_embeddings.py

@@ -14,6 +14,7 @@ import sys
 
 import numpy as np
 import pyarrow as pa
+
 from lancedb.embeddings import with_embeddings
 
 

python/tests/test_fts.py

@@ -13,13 +13,13 @@
 import os
 import random
 
-import lancedb.fts
 import numpy as np
 import pandas as pd
 import pytest
 import tantivy
 
 import lancedb as ldb
+import lancedb.fts
 
 
 @pytest.fixture

python/tests/test_io.py

@@ -1,49 +0,0 @@
-#  Copyright 2023 LanceDB Developers
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-import os
-import pytest
-
-import lancedb
-
-# You need to setup AWS credentials an a base path to run this test. Example
-#    AWS_PROFILE=default TEST_S3_BASE_URL=s3://my_bucket/dataset pytest tests/test_io.py
-
-@pytest.mark.skipif(
-    (os.environ.get("TEST_S3_BASE_URL") is None),
-    reason="please setup s3 base url",
-)
-def test_s3_io():
-    db = lancedb.connect(os.environ.get("TEST_S3_BASE_URL"))
-    assert db.table_names() == []
-
-    table = db.create_table(
-        "test",
-        data=[
-            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
-        ],
-    )
-    rs = table.search([100, 100]).limit(1).to_df()
-    assert len(rs) == 1
-    assert rs["item"].iloc[0] == "bar"
-
-    rs = table.search([100, 100]).where("price < 15").limit(2).to_df()
-    assert len(rs) == 1
-    assert rs["item"].iloc[0] == "foo"
-
-    assert db.table_names() == ["test"]
-    assert "test" in db
-    assert len(db) == 1
-
-    assert db.open_table("test").name == db["test"].name

python/tests/test_query.py

@@ -17,6 +17,7 @@ import pandas as pd
 import pandas.testing as tm
 import pyarrow as pa
 import pytest
+
 from lancedb.query import LanceQueryBuilder
 
 

python/tests/test_table.py

@@ -16,6 +16,7 @@ from pathlib import Path
 import pandas as pd
 import pyarrow as pa
 import pytest
+
 from lancedb.table import LanceTable
 
 

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -15,7 +15,7 @@ arrow-ipc = "37.0"
 arrow-schema = "37.0"
 once_cell = "1"
 futures = "0.3"
-lance = "0.4.17"
+lance = "0.4.3"
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

rust/ffi/node/src/index/vector.rs

@@ -39,7 +39,7 @@ pub(crate) fn table_create_vector_index(mut cx: FunctionContext) -> JsResult<JsP
         let add_result = table
             .lock()
             .unwrap()
-            .create_index(&index_params_builder)
+            .create_idx(&index_params_builder)
             .await;
 
         deferred.settle_with(&channel, move |mut cx| {

rust/ffi/node/src/lib.rs

@@ -56,46 +56,23 @@ fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
     RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
 }
 
-fn database_new(mut cx: FunctionContext) -> JsResult<JsPromise> {
+fn database_new(mut cx: FunctionContext) -> JsResult<JsBox<JsDatabase>> {
     let path = cx.argument::<JsString>(0)?.value(&mut cx);
-
-    let rt = runtime(&mut cx)?;
-    let channel = cx.channel();
-    let (deferred, promise) = cx.promise();
-
-    rt.spawn(async move {
-        let database = Database::connect(&path).await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let db = JsDatabase {
-                database: Arc::new(database.or_else(|err| cx.throw_error(err.to_string()))?),
-            };
-            Ok(cx.boxed(db))
-        });
-    });
-    Ok(promise)
+    let db = JsDatabase {
+        database: Arc::new(Database::connect(path).or_else(|err| cx.throw_error(err.to_string()))?),
+    };
+    Ok(cx.boxed(db))
 }
 
-fn database_table_names(mut cx: FunctionContext) -> JsResult<JsPromise> {
+fn database_table_names(mut cx: FunctionContext) -> JsResult<JsArray> {
     let db = cx
         .this()
         .downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
-
-    let rt = runtime(&mut cx)?;
-    let (deferred, promise) = cx.promise();
-    let channel = cx.channel();
-    let database = db.database.clone();
-
-    rt.spawn(async move {
-        let tables_rst = database.table_names().await;
-
-        deferred.settle_with(&channel, move |mut cx| {
-            let tables = tables_rst.or_else(|err| cx.throw_error(err.to_string()))?;
-            let table_names = convert::vec_str_to_array(&tables, &mut cx);
-            table_names
-        });
-    });
-    Ok(promise)
+    let tables = db
+        .database
+        .table_names()
+        .or_else(|err| cx.throw_error(err.to_string()))?;
+    convert::vec_str_to_array(&tables, &mut cx)
 }
 
 fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
@@ -110,7 +87,7 @@ fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     let (deferred, promise) = cx.promise();
     rt.spawn(async move {
-        let table_rst = database.open_table(&table_name).await;
+        let table_rst = database.open_table(table_name).await;
 
         deferred.settle_with(&channel, move |mut cx| {
             let table = Arc::new(Mutex::new(
@@ -209,7 +186,7 @@ fn table_create(mut cx: FunctionContext) -> JsResult<JsPromise> {
 
     rt.block_on(async move {
         let batch_reader: Box<dyn RecordBatchReader> = Box::new(RecordBatchBuffer::new(batches));
-        let table_rst = database.create_table(&table_name, batch_reader).await;
+        let table_rst = database.create_table(table_name, batch_reader).await;
 
         deferred.settle_with(&channel, move |mut cx| {
             let table = Arc::new(Mutex::new(

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -12,9 +12,7 @@ repository = "https://github.com/lancedb/lancedb"
 arrow-array = "37.0"
 arrow-data = "37.0"
 arrow-schema = "37.0"
-object_store = "0.5.6"
-snafu = "0.7.4"
-lance = "0.4.17"
+lance = "0.4.3"
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
 
 [dev-dependencies]

rust/vectordb/src/database.rs

@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fs::create_dir_all;
-use std::path::Path;
-
 use arrow_array::RecordBatchReader;
-use lance::io::object_store::ObjectStore;
-use snafu::prelude::*;
+use std::fs::create_dir_all;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
-use crate::error::{CreateDirSnafu, Result};
+use crate::error::Result;
 use crate::table::Table;
 
 pub struct Database {
-    object_store: ObjectStore,
-
-    pub(crate) uri: String,
+    pub(crate) path: Arc<PathBuf>,
 }
 
 const LANCE_EXTENSION: &str = "lance";
@@ -41,24 +37,13 @@ impl Database {
     /// # Returns
     ///
     /// * A [Database] object.
-    pub async fn connect(uri: &str) -> Result<Database> {
-        let object_store = ObjectStore::new(uri).await?;
-        if object_store.is_local() {
-            Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
-        }
-        Ok(Database {
-            uri: uri.to_string(),
-            object_store,
-        })
-    }
-
-    /// Try to create a local directory to store the lancedb dataset
-    fn try_create_dir(path: &str) -> core::result::Result<(), std::io::Error> {
-        let path = Path::new(path);
-        if !path.try_exists()? {
+    pub fn connect<P: AsRef<Path>>(path: P) -> Result<Database> {
+        if !path.as_ref().try_exists()? {
             create_dir_all(&path)?;
         }
-        Ok(())
+        Ok(Database {
+            path: Arc::new(path.as_ref().to_path_buf()),
+        })
     }
 
     /// Get the names of all tables in the database.
@@ -66,13 +51,12 @@ impl Database {
     /// # Returns
     ///
     /// * A [Vec<String>] with all table names.
-    pub async fn table_names(&self) -> Result<Vec<String>> {
+    pub fn table_names(&self) -> Result<Vec<String>> {
         let f = self
-            .object_store
-            .read_dir("/")
-            .await?
-            .iter()
-            .map(|fname| Path::new(fname))
+            .path
+            .read_dir()?
+            .flatten()
+            .map(|dir_entry| dir_entry.path())
             .filter(|path| {
                 let is_lance = path
                     .extension()
@@ -92,10 +76,10 @@ impl Database {
 
     pub async fn create_table(
         &self,
-        name: &str,
+        name: String,
         batches: Box<dyn RecordBatchReader>,
     ) -> Result<Table> {
-        Table::create(&self.uri, name, batches).await
+        Table::create(self.path.clone(), name, batches).await
     }
 
     /// Open a table in the database.
@@ -106,8 +90,8 @@ impl Database {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open_table(&self, name: &str) -> Result<Table> {
-        Table::open(&self.uri, name).await
+    pub async fn open_table(&self, name: String) -> Result<Table> {
+        Table::open(self.path.clone(), name).await
     }
 }
 
@@ -121,10 +105,10 @@ mod tests {
     #[tokio::test]
     async fn test_connect() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = Database::connect(uri).await.unwrap();
+        let path_buf = tmp_dir.into_path();
+        let db = Database::connect(&path_buf);
 
-        assert_eq!(db.uri, uri);
+        assert_eq!(db.unwrap().path.as_path(), path_buf.as_path())
     }
 
     #[tokio::test]
@@ -134,16 +118,10 @@ mod tests {
         create_dir_all(tmp_dir.path().join("table2.lance")).unwrap();
         create_dir_all(tmp_dir.path().join("invalidlance")).unwrap();
 
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = Database::connect(uri).await.unwrap();
-        let tables = db.table_names().await.unwrap();
+        let db = Database::connect(&tmp_dir.into_path()).unwrap();
+        let tables = db.table_names().unwrap();
         assert_eq!(tables.len(), 2);
         assert!(tables.contains(&String::from("table1")));
         assert!(tables.contains(&String::from("table2")));
     }
-
-    #[tokio::test]
-    async fn test_connect_s3() {
-        // let db = Database::connect("s3://bucket/path/to/database").await.unwrap();
-    }
 }

rust/vectordb/src/error.rs

@@ -12,50 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use snafu::Snafu;
-
-#[derive(Debug, Snafu)]
-#[snafu(visibility(pub(crate)))]
+#[derive(Debug)]
 pub enum Error {
-    #[snafu(display("LanceDBError: Invalid table name: {name}"))]
-    InvalidTableName { name: String },
-    #[snafu(display("LanceDBError: Table '{name}' was not found"))]
-    TableNotFound { name: String },
-    #[snafu(display("LanceDBError: Table '{name}' already exists"))]
-    TableAlreadyExists { name: String },
-    #[snafu(display("LanceDBError: Unable to created lance dataset at {path}: {source}"))]
-    CreateDir {
-        path: String,
-        source: std::io::Error,
-    },
-    #[snafu(display("LanceDBError: {message}"))]
-    Store { message: String },
-    #[snafu(display("LanceDBError: {message}"))]
-    Lance { message: String },
+    IO(String),
+    Lance(String),
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let (catalog, message) = match self {
+            Self::IO(s) => ("I/O", s.as_str()),
+            Self::Lance(s) => ("Lance", s.as_str()),
+        };
+        write!(f, "LanceDBError({catalog}): {message}")
+    }
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
 
+impl From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Self {
+        Self::IO(e.to_string())
+    }
+}
+
 impl From<lance::Error> for Error {
     fn from(e: lance::Error) -> Self {
-        Self::Lance {
-            message: e.to_string(),
-        }
-    }
-}
-
-impl From<object_store::Error> for Error {
-    fn from(e: object_store::Error) -> Self {
-        Self::Store {
-            message: e.to_string(),
-        }
-    }
-}
-
-impl From<object_store::path::Error> for Error {
-    fn from(e: object_store::path::Error) -> Self {
-        Self::Store {
-            message: e.to_string(),
-        }
+        Self::Lance(e.to_string())
     }
 }

rust/vectordb/src/table.rs

@@ -12,35 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::path::Path;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use arrow_array::{Float32Array, RecordBatchReader};
 use lance::dataset::{Dataset, WriteMode, WriteParams};
 use lance::index::IndexType;
-use snafu::prelude::*;
 
-use crate::error::{Error, InvalidTableNameSnafu, Result};
+use crate::error::{Error, Result};
 use crate::index::vector::VectorIndexBuilder;
 use crate::query::Query;
 
 pub const VECTOR_COLUMN_NAME: &str = "vector";
+
 pub const LANCE_FILE_EXTENSION: &str = "lance";
 
 /// A table in a LanceDB database.
-#[derive(Debug)]
 pub struct Table {
     name: String,
-    uri: String,
+    path: String,
     dataset: Arc<Dataset>,
 }
 
-impl std::fmt::Display for Table {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Table({})", self.name)
-    }
-}
-
 impl Table {
     /// Opens an existing Table
     ///
@@ -52,28 +45,18 @@ impl Table {
     /// # Returns
     ///
     /// * A [Table] object.
-    pub async fn open(base_uri: &str, name: &str) -> Result<Self> {
-        let path = Path::new(base_uri);
-
-        let table_uri = path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
+    pub async fn open(base_path: Arc<PathBuf>, name: String) -> Result<Self> {
+        let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+        let ds_uri = ds_path
             .to_str()
-            .context(InvalidTableNameSnafu { name })?;
-
-        let dataset = Dataset::open(&uri).await.map_err(|e| match e {
-            lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
-                name: name.to_string(),
-            },
-            e => Error::Lance {
-                message: e.to_string(),
-            },
-        })?;
-        Ok(Table {
-            name: name.to_string(),
-            uri: uri.to_string(),
+            .ok_or(Error::IO(format!("Unable to find table {}", name)))?;
+        let dataset = Dataset::open(ds_uri).await?;
+        let table = Table {
+            name,
+            path: ds_uri.to_string(),
             dataset: Arc::new(dataset),
-        })
+        };
+        Ok(table)
     }
 
     /// Creates a new Table
@@ -88,36 +71,25 @@ impl Table {
     ///
     /// * A [Table] object.
     pub async fn create(
-        base_uri: &str,
-        name: &str,
+        base_path: Arc<PathBuf>,
+        name: String,
         mut batches: Box<dyn RecordBatchReader>,
     ) -> Result<Self> {
-        let base_path = Path::new(base_uri);
-        let table_uri = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
-        let uri = table_uri
-            .as_path()
+        let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
+        let path = ds_path
             .to_str()
-            .context(InvalidTableNameSnafu { name })?
-            .to_string();
-        let dataset = Dataset::write(&mut batches, &uri, Some(WriteParams::default()))
-            .await
-            .map_err(|e| match e {
-                lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
-                    name: name.to_string(),
-                },
-                e => Error::Lance {
-                    message: e.to_string(),
-                },
-            })?;
+            .ok_or(Error::IO(format!("Unable to find table {}", name)))?;
+
+        let dataset =
+            Arc::new(Dataset::write(&mut batches, path, Some(WriteParams::default())).await?);
         Ok(Table {
-            name: name.to_string(),
-            uri,
-            dataset: Arc::new(dataset),
+            name,
+            path: path.to_string(),
+            dataset,
         })
     }
 
-    /// Create index on the table.
-    pub async fn create_index(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
+    pub async fn create_idx(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
         use lance::index::DatasetIndexExt;
 
         let dataset = self
@@ -153,7 +125,8 @@ impl Table {
         let mut params = WriteParams::default();
         params.mode = write_mode.unwrap_or(WriteMode::Append);
 
-        self.dataset = Arc::new(Dataset::write(&mut batches, &self.uri, Some(params)).await?);
+        self.dataset =
+            Arc::new(Dataset::write(&mut batches, self.path.as_str(), Some(params)).await?);
         Ok(batches.count())
     }
 
@@ -178,8 +151,6 @@ impl Table {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
     use arrow_array::{
         Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, RecordBatchReader,
     };
@@ -190,68 +161,53 @@ mod tests {
     use lance::index::vector::ivf::IvfBuildParams;
     use lance::index::vector::pq::PQBuildParams;
     use rand::Rng;
+    use std::sync::Arc;
     use tempfile::tempdir;
 
-    use super::*;
+    use crate::error::Result;
     use crate::index::vector::IvfPQIndexBuilder;
+    use crate::table::Table;
+
+    #[tokio::test]
+    async fn test_new_table_not_exists() {
+        let tmp_dir = tempdir().unwrap();
+        let path_buf = tmp_dir.into_path();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string()).await;
+        assert!(table.is_err());
+    }
 
     #[tokio::test]
     async fn test_open() {
         let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        Dataset::write(
+            &mut batches,
+            path_buf.join("test.lance").to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string())
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
-
         assert_eq!(table.name, "test")
     }
 
-    #[tokio::test]
-    async fn test_open_not_found() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let table = Table::open(uri, "test").await;
-        assert!(matches!(table.unwrap_err(), Error::TableNotFound { .. }));
-    }
-
-    #[test]
-    fn test_object_store_path() {
-        use std::path::Path as StdPath;
-        let p = StdPath::new("s3://bucket/path/to/file");
-        let c = p.join("subfile");
-        assert_eq!(c.to_str().unwrap(), "s3://bucket/path/to/file/subfile");
-    }
-
-    #[tokio::test]
-    async fn test_create_already_exists() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        let schema = batches.schema().clone();
-        Table::create(&uri, "test", batches).await.unwrap();
-
-        let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        let result = Table::create(&uri, "test", batches).await;
-        assert!(matches!(
-            result.unwrap_err(),
-            Error::TableAlreadyExists { .. }
-        ));
-    }
-
     #[tokio::test]
     async fn test_add() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
         let schema = batches.schema().clone();
-        let mut table = Table::create(&uri, "test", batches).await.unwrap();
+        let mut table = Table::create(Arc::new(path_buf), "test".to_string(), batches)
+            .await
+            .unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
         let new_batches: Box<dyn RecordBatchReader> =
@@ -269,11 +225,13 @@ mod tests {
     #[tokio::test]
     async fn test_add_overwrite() {
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
         let schema = batches.schema().clone();
-        let mut table = Table::create(uri, "test", batches).await.unwrap();
+        let mut table = Table::create(Arc::new(path_buf), "test".to_string(), batches)
+            .await
+            .unwrap();
         assert_eq!(table.count_rows().await.unwrap(), 10);
 
         let new_batches: Box<dyn RecordBatchReader> =
@@ -294,16 +252,21 @@ mod tests {
     #[tokio::test]
     async fn test_search() {
         let tmp_dir = tempdir().unwrap();
-        let dataset_path = tmp_dir.path().join("test.lance");
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
-        Dataset::write(&mut batches, dataset_path.to_str().unwrap(), None)
+        Dataset::write(
+            &mut batches,
+            path_buf.join("test.lance").to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+
+        let table = Table::open(Arc::new(path_buf), "test".to_string())
             .await
             .unwrap();
 
-        let table = Table::open(uri, "test").await.unwrap();
-
         let vector = Float32Array::from_iter_values([0.1, 0.2]);
         let query = table.search(vector.clone());
         assert_eq!(vector, query.query_vector);
@@ -328,7 +291,7 @@ mod tests {
         use arrow_array::Float32Array;
 
         let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
+        let path_buf = tmp_dir.into_path();
 
         let dimension = 16;
         let schema = Arc::new(ArrowSchema::new(vec![Field::new(
@@ -355,7 +318,9 @@ mod tests {
         .unwrap()]);
 
         let reader: Box<dyn RecordBatchReader + Send> = Box::new(batches);
-        let mut table = Table::create(uri, "test", reader).await.unwrap();
+        let mut table = Table::create(Arc::new(path_buf), "test".to_string(), reader)
+            .await
+            .unwrap();
 
         let mut i = IvfPQIndexBuilder::new();
 
@@ -365,7 +330,7 @@ mod tests {
             .ivf_params(IvfBuildParams::new(256))
             .pq_params(PQBuildParams::default());
 
-        table.create_index(index_builder).await.unwrap();
+        table.create_idx(index_builder).await.unwrap();
 
         assert_eq!(table.dataset.load_indices().await.unwrap().len(), 1);
         assert_eq!(table.count_rows().await.unwrap(), 512);