bump version for v0.1.5-python

Co-authored-by: Will Jones <willjones127@gmail.com>

.github/workflows/make_release_commit.yml

@@ -1,48 +0,0 @@
-name: Create release commit
-
-on:
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: 'Just create the local commit/tags but do not push it'
-        required: true
-        default: "false"
-        type: choice
-        options:
-          - "true"
-          - "false"
-      part:
-        description: 'What kind of release is this?'
-        required: true
-        default: 'patch'
-        type: choice
-        options:
-          - patch
-          - minor
-          - major
-
-jobs:
-  bump-version:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out main
-      uses: actions/checkout@v3
-      with:
-        ref: main
-        persist-credentials: false
-        fetch-depth: 0
-        lfs: true
-    - name: Install cargo utils
-      run: cargo install cargo-edit
-    - name: Bump versions
-      run: |
-        NEW_VERSION=$(bash ci/bump_versions.sh ${{ inputs.part }})
-        echo "New version: v$NEW_VERSION"
-        git tag v$NEW_VERSION
-    - name: Push new version and tag
-      if: ${{ inputs.dry_run }} == "false"
-      uses: ad-m/github-push-action@master
-      with:
-        github_token: ${{ secrets.RELEASE_TOKEN }}
-        branch: main
-        tags: true

.github/workflows/node.yml

@@ -67,12 +67,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
-        npm run pack-build
+        npm run tsc
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: npm run test
   macos:
@@ -98,12 +94,8 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run tsc
         npm run build
-        npm run pack-build
+        npm run tsc
-        npm install --no-save ./dist/lancedb-vectordb-*.tgz
-        # Remove index.node to test with dependency installed
-        rm index.node
     - name: Test
       run: |
         npm run test

.github/workflows/release.yml

@@ -1,167 +0,0 @@
-name: Prepare Release
-
-# NOTE: Python is a separate release for now.
-
-# Currently disabled until it can be completed.
-# on:
-#   push:
-#     tags:
-#       - v*
-
-jobs:
-  draft-release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          prerelease: true # hardcoded on for now
-          generate_release_notes: true
-  
-  rust:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: rust/vectordb
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Package Rust
-        run: cargo package --all-features
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: target/package/vectordb-*.crate
-          fail_on_unmatched_files: true
-
-  node:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20
-          cache: 'npm'
-          cache-dependency-path: node/package-lock.json
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y protobuf-compiler libssl-dev
-      - name: Build
-        run: |
-          npm ci
-          npm run tsc
-          npm pack
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: node/vectordb-*.tgz
-          fail_on_unmatched_files: true
-
-  node-macos:
-    runs-on: macos-12
-    needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v33
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install npm dependencies
-      run: |
-        cd node
-        npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    - uses: softprops/action-gh-release@v1
-      with:
-        draft: true
-        files: node/dist/lancedb-vectordb-darwin*.tgz
-        fail_on_unmatched_files: true
-
-  node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
-    runs-on: ubuntu-latest
-    needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        libc:
-          - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
-          # Right now we have to build node from source which is too expensive.
-          # - musl
-        arch:
-          - x86_64
-          # Building on aarch64 is too slow for now
-          # - aarch64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Change owner to root (for npm)
-      # The docker container is run as root, so we need the files to be owned by root
-      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
-      run: sudo chown -R root:root .
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2  
-      with:
-        platforms: arm64
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
-      run: |
-        docker run \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
-      run: |
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    - uses: softprops/action-gh-release@v1
-      with:
-        draft: true
-        files: node/dist/lancedb-vectordb-linux*.tgz
-        fail_on_unmatched_files: true
-
-  release:
-    needs: [rust, node, node-macos, node-linux]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v3
-      - name: Publish to NPM
-        run: |
-          for filename in node/dist/*.tgz; do
-            npm publish --dry-run $filename
-          done
-      - name: Publish to crates.io
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: |
-          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
-      # - uses: softprops/action-gh-release@v1
-      #   with:
-      #     draft: false

.gitignore

@@ -4,8 +4,6 @@
 **/__pycache__
 .DS_Store
 
-.vscode
-
 rust/target
 rust/Cargo.lock
 

Cargo.lock

@@ -3358,7 +3358,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vectordb"
-version = "0.1.2"
+version = "0.0.1"
 dependencies = [
  "arrow-array",
  "arrow-data",
@@ -3373,7 +3373,7 @@ dependencies = [
 
 [[package]]
 name = "vectordb-node"
-version = "0.1.2"
+version = "0.1.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

ci/build_linux_artifacts.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-# Builds the Linux artifacts (node binaries).
-# Usage: ./build_linux_artifacts.sh [target]
-# Targets supported: 
-# - x86_64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-gnu:centos
-# - aarch64-unknown-linux-musl
-# - x86_64-unknown-linux-musl
-
-# TODO: refactor this into a Docker container we can pull
-
-set -e
-
-setup_dependencies() {
-    echo "Installing system dependencies..."
-    if [[ $1 == *musl ]]; then
-        # musllinux
-        apk add openssl-dev
-    else
-        # manylinux2014
-        yum install -y openssl-devel unzip
-    fi
-
-    if [[ $1 == x86_64* ]]; then
-        ARCH=x86_64
-    else
-        # gnu target
-        ARCH=aarch_64
-    fi
-    
-    # Install new enough protobuf (yum-provided is old)
-    PB_REL=https://github.com/protocolbuffers/protobuf/releases
-    PB_VERSION=23.1
-    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
-}
-
-install_node() {
-    echo "Installing node..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-    source "$HOME"/.bashrc
-
-    if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s --no-progress 17
-    else
-        nvm install --no-progress 17 # latest that supports glibc 2.17
-    fi
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
-    pushd node
-
-    npm ci
-    
-    if [[ $1 == *musl ]]; then
-        # This is needed for cargo to allow build cdylibs with musl
-        export RUSTFLAGS="-C target-feature=-crt-static"
-    fi
-
-    # Cargo can run out of memory while pulling dependencies, espcially when running
-    # in QEMU. This is a workaround for that.
-    export CARGO_NET_GIT_FETCH_WITH_CLI=true
-
-    # We don't pass in target, since the native target here already matches
-    # and openblas-src doesn't do well with cross-compilation.
-    npm run build-release
-    npm run pack-build
-
-    popd
-}
-
-TARGET=${1:-x86_64-unknown-linux-gnu}
-# Others:
-# aarch64-unknown-linux-gnu
-# x86_64-unknown-linux-musl
-# aarch64-unknown-linux-musl
-
-setup_dependencies $TARGET
-install_node $TARGET
-install_rust
-build_node_binary $TARGET

ci/build_macos_artifacts.sh

@@ -1,33 +0,0 @@
-# Builds the macOS artifacts (node binaries).
-# Usage: ./ci/build_macos_artifacts.sh [target]
-# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
-
-prebuild_rust() {
-    # Building here for the sake of easier debugging.
-    pushd rust/ffi/node
-    echo "Building rust library for $1"
-    export RUST_BACKTRACE=1
-    cargo build --release --target $1
-    popd
-}
-
-build_node_binaries() {
-    pushd node
-    echo "Building node library for $1"
-    npm run build-release -- --target $1
-    npm run pack-build -- --target $1
-    popd
-}
-
-if [ -n "$1" ]; then
-    targets=$1
-else
-    targets="x86_64-apple-darwin aarch64-apple-darwin"
-fi
-
-echo "Building artifacts for targets: $targets"
-for target in $targets
-    do
-    prebuild_rust $target
-    build_node_binaries $target
-done

ci/bump_versions.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-set -e
-
-# if cargo bump isn't installed return an error
-if ! cargo set-version &> /dev/null
-then
-    echo "cargo-edit could not be found. Install with `cargo install cargo-edit`"
-    exit
-fi
-
-BUMP_PART=${1:-patch}
-
-# if BUMP_PART isn't patch, minor, or major return an error
-if [ "$BUMP_PART" != "patch" ] && [ "$BUMP_PART" != "minor" ] && [ "$BUMP_PART" != "major" ]
-then
-    echo "BUMP_PART must be one of patch, minor, or major"
-    exit
-fi
-
-function get_crate_version() {
-    cargo pkgid -p $1 | cut -d@ -f2 | cut -d# -f2
-}
-
-# First, validate versions are starting as same
-VECTORDB_VERSION=$(get_crate_version vectordb)
-FFI_NODE_VERSION=$(get_crate_version vectordb-node)
-
-# FYI, we pipe all output to /dev/null because the only thing we want to ouput
-# if success is the new tag. This way it can be then used with `git tag`.
-pushd node > /dev/null
-NODE_VERSION=$(npm pkg get version | xargs echo)
-popd > /dev/null
-
-if [ "$VECTORDB_VERSION" != "$FFI_NODE_VERSION" ] || [ "$VECTORDB_VERSION" != "$NODE_VERSION" ]
-then
-    echo "Version mismatch between rust/vectordb, rust/ffi/node, and node"
-    echo "rust/vectordb: $VECTORDB_VERSION"
-    echo "rust/ffi/node: $FFI_NODE_VERSION"
-    echo "node: $NODE_VERSION"
-    exit
-fi
-
-cargo set-version --bump $BUMP_PART > /dev/null 2>&1
-NEW_VERSION=$(get_crate_version vectordb)
-
-pushd node > /dev/null
-npm version $BUMP_PART > /dev/null
-
-# Also need to update version of the native modules
-NATIVE_MODULES=$(npm pkg get optionalDependencies | jq 'keys[]' | grep @vectordb/ | tr -d '"')
-for module in $NATIVE_MODULES
-do
-    npm install $module@$NEW_VERSION --save-optional > /dev/null
-done
-popd > /dev/null
-
-
-echo $NEW_VERSION

ci/release_process.md

@@ -1,136 +0,0 @@
-# How to release
-
-This is for the Rust crate and Node module. For now, the Python module is
-released separately.
-
-<!--
-The release is started by bumping the versions and pushing a new tag. To do this
-automatically, use the `make_release_commit` GitHub action.
-
-When the tag is pushed, GitHub actions will start building the libraries and
-will upload them to a draft release.
-
-While those jobs are running, edit the release notes as needed. For example, 
-bring relevant new features and bugfixes to the top of the notes and the testing
-and CI changes to the bottom.
-
-Once the jobs have finished, the release will be marked as not draft and the
-artifacts will be released to crates.io, NPM, and PyPI.
-
--->
-
-## Manual process
-
-The manual release process can be completed on a MacOS machine.
-
-### Bump the versions
-
-You can use the script `ci/bump_versions.sh` to bump the versions. It defaults
-to a `patch` bump, but you can also pass `minor` and `major`. Once you have the
-tag created, push it to GitHub.
-
-```shell
-VERSION=$(bash ci/bump_versions.sh)
-git tag v$VERSION
-git push origin v$VERSION
-```
-
-### Build the MacOS release libraries
-
-One-time setup:
-
-```shell
-rustup target add x86_64-apple-darwin aarch64-apple-darwin
-```
-
-To build both x64 and arm64, run `ci/build_macos_artifacts.sh` without any args:
-
-```shell
-bash ci/build_macos_artifacts.sh
-```
-
-### Build the Linux release libraries
-
-To build a Linux library, we need to use docker with a different build script:
-
-```shell
-ARCH=aarch64
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
-```
-
-For x64, change `ARCH` to `x86_64`. NOTE: compiling for a different architecture
-than your machine in Docker is very slow. It's best to do this on a machine with
-matching architecture.
-
-
-<!--
-Similar script for musl binaries (not yet working):
-
-```shell
-ARCH=aarch64
-docker run \
-    --user $(id -u) \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
-```
-
--->
-
-<!--
-
-For debugging, use these snippets:
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
-    bash
-```
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash
-```
-
-Note: musllinux_1_1 is Alpine Linux 3.12
--->
-
-
-### Build the npm module
-
-To build the typescript and create a release tarball, run:
-
-```shell
-npm ci
-npm tsc
-npm pack
-```
-
-### Release to npm
-
-Assuming you still have `VERSION` set from earlier:
-
-```shell
-pushd node
-npm publish lancedb-vectordb-$VERSION.tgz
-for tarball in ./dist/lancedb-vectordb-*-$VERSION.tgz;
-    do 
-    npm publish $tarball
-done
-popd
-```
-
-### Release to crates.io
-
-```shell
-cargo publish -p vectordb
-cargo publish -p vectordb-node
-```

node/.npmignore

@@ -1,4 +0,0 @@
-gen_test_data.py
-index.node
-dist/lancedb*.tgz
-vectordb*.tgz

node/README.md

@@ -8,10 +8,6 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
-This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
-
 ## Usage
 
 ### Basic Example
@@ -28,34 +24,18 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
-To build everything fresh:
+The LanceDB javascript is built with npm:
 
 ```bash
-npm install
 npm run tsc
-npm run build
 ```
 
-Then you should be able to run the tests with:
+Run the tests with
 
 ```bash
 npm test
 ```
 
-### Rebuilding Rust library
-
-```bash
-npm run build
-```
-
-### Rebuilding Typescript
-
-```bash
-npm run tsc
-```
-
-### Fix lints
-
 To run the linter and have it automatically fix all errors
 
 ```bash

node/native.js

@@ -12,26 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-const { currentTarget } = require('@neon-rs/load');
-
 let nativeLib;
 
-try {
+function getPlatformLibrary() {
-    nativeLib = require(`@lancedb/vectordb-${currentTarget()}`);
+    if (process.platform === "darwin" && process.arch == "arm64") {
-} catch (e) {
+        return require('./aarch64-apple-darwin.node');
-    try {
+    } else if (process.platform === "darwin" && process.arch == "x64") {
-        // Might be developing locally, so try that. But don't expose that error
+        return require('./x86_64-apple-darwin.node');
-        // to the user.
+    } else if (process.platform === "linux" && process.arch == "x64") {
-        nativeLib = require("./index.node");
+        return require('./x86_64-unknown-linux-gnu.node');
-    } catch {
+    } else {
-        throw new Error(`vectordb: failed to load native library.
+        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-  You may need to run \`npm install @lancedb/vectordb-${currentTarget()}\`.
-      
-  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
-      
-  Source error: ${e}`);
     }
 }
 
-// Dynamic require for runtime.
+try {
-module.exports = nativeLib;
+    nativeLib = require('./index.node')
+} catch (e) {
+    if (e.code === "MODULE_NOT_FOUND") {
+        nativeLib = getPlatformLibrary();
+    } else {
+        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
+    }
+}
+
+module.exports = nativeLib
+

node/package-lock.json

@@ -1,31 +1,22 @@
 {
   "name": "vectordb",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.1.2",
+      "version": "0.1.3",
-      "cpu": [
-        "x64",
-        "arm64"
-      ],
       "license": "Apache-2.0",
-      "os": [
-        "darwin",
-        "linux"
-      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
-        "@neon-rs/load": "^0.0.74",
         "apache-arrow": "^12.0.0"
       },
       "devDependencies": {
-        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
+        "@types/sinon": "^10.0.15",
         "@types/temp": "^0.9.1",
         "@typescript-eslint/eslint-plugin": "^5.59.1",
         "cargo-cp-artifact": "^0.1",
@@ -42,12 +33,6 @@
         "ts-node": "^10.9.1",
         "ts-node-dev": "^2.0.0",
         "typescript": "*"
-      },
-      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.1.2",
-        "@lancedb/vectordb-darwin-x64": "0.1.2",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.1.2",
-        "@lancedb/vectordb-linux-x64-gnu": "0.1.2"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -215,46 +200,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.2",
-      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-darwin-arm64/0.1.2/84d71331e03e8aaeb9fb12cdacc759dc82cfd3b0",
-      "integrity": "sha512-DU6tHmmn/coSj5r5FGwTMXMQfsSSxQN1ozOl9mFUXr0aVtlx5nlA8ZY5BAF/V371yL5QzNPKtaNpogP6iw51NA==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.2",
-      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-linux-arm64-gnu/0.1.2/d5a9d66c3969494cf3546195fb5511f9f49aa295",
-      "integrity": "sha512-LZZ4KgoGqD5AzKX/utBrsxrwXq6whpUNa02tWxl/ND/601ruNi9ZUaXCTb1rSVUWJkgMR2wASk15kssyaPRSjw==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@neon-rs/cli": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
-      "dev": true,
-      "bin": {
-        "neon": "index.js"
-      }
-    },
-    "node_modules/@neon-rs/load": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
-      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
-    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4497,29 +4442,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.1.2",
-      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-darwin-arm64/0.1.2/84d71331e03e8aaeb9fb12cdacc759dc82cfd3b0",
-      "integrity": "sha512-DU6tHmmn/coSj5r5FGwTMXMQfsSSxQN1ozOl9mFUXr0aVtlx5nlA8ZY5BAF/V371yL5QzNPKtaNpogP6iw51NA==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.1.2",
-      "resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb-linux-arm64-gnu/0.1.2/d5a9d66c3969494cf3546195fb5511f9f49aa295",
-      "integrity": "sha512-LZZ4KgoGqD5AzKX/utBrsxrwXq6whpUNa02tWxl/ND/601ruNi9ZUaXCTb1rSVUWJkgMR2wASk15kssyaPRSjw==",
-      "optional": true
-    },
-    "@neon-rs/cli": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
-      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
-      "dev": true
-    },
-    "@neon-rs/load": {
-      "version": "0.0.74",
-      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
-      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
-    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

node/package.json

@@ -1,18 +1,15 @@
 {
   "name": "vectordb",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
     "build-release": "npm run build -- --release",
-    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts",
+    "lint": "eslint src --ext .js,.ts"
-    "pack-build": "neon pack-build",
-    "check-npm": "printenv && which node && which npm && npm --version"
   },
   "repository": {
     "type": "git",
@@ -27,7 +24,6 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
-    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
@@ -51,29 +47,6 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
-    "@neon-rs/load": "^0.0.74",
     "apache-arrow": "^12.0.0"
-  },
-  "os": [
-    "darwin",
-    "linux"
-  ],
-  "cpu": [
-    "x64",
-    "arm64"
-  ],
-  "neon": {
-    "targets": {
-      "x86_64-apple-darwin": "@lancedb/vectordb-darwin-x64",
-      "aarch64-apple-darwin": "@lancedb/vectordb-darwin-arm64",
-      "x86_64-unknown-linux-gnu": "@lancedb/vectordb-linux-x64-gnu",
-      "aarch64-unknown-linux-gnu": "@lancedb/vectordb-linux-arm64-gnu"
-    }
-  },
-  "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.1.2",
-    "@lancedb/vectordb-darwin-x64": "0.1.2",
-    "@lancedb/vectordb-linux-x64-gnu": "0.1.2",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.1.2"
   }
 }

python/lancedb/db.py

@@ -13,13 +13,16 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
+import os
 
 import pyarrow as pa
+from pyarrow import fs
 
 from .common import DATA, URI
 from .table import LanceTable
-from .util import get_uri_scheme
+from .util import get_uri_scheme, get_uri_location
 
 
 class LanceDBConnection:
@@ -47,11 +50,20 @@ class LanceDBConnection:
         -------
         A list of table names.
         """
-        if get_uri_scheme(self.uri) == "file":
+        try:
-            return [p.stem for p in Path(self.uri).glob("*.lance")]
+            filesystem, path = fs.FileSystem.from_uri(self.uri)
-        raise NotImplementedError(
+        except pa.ArrowInvalid:
-            "List table_names is only supported for local filesystem for now"
+            raise NotImplementedError(
-        )
+                "Unsupported scheme: " + self.uri
+            )
+
+        try:
+            paths = filesystem.get_file_info(fs.FileSelector(get_uri_location(self.uri)))
+        except FileNotFoundError:
+            # It is ok if the file does not exist since it will be created
+            paths = []
+        tables = [os.path.splitext(file_info.base_name)[0] for file_info in paths if file_info.extension == 'lance']
+        return tables
 
     def __len__(self) -> int:
         return len(self.table_names())
@@ -112,3 +124,15 @@ class LanceDBConnection:
         A LanceTable object representing the table.
         """
         return LanceTable(self, name)
+
+    def drop_table(self, name: str):
+        """Drop a table from the database.
+
+        Parameters
+        ----------
+        name: str
+            The name of the table.
+        """
+        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        table_path = os.path.join(path, name + ".lance")
+        filesystem.delete_dir(table_path)

python/lancedb/util.py

@@ -41,3 +41,23 @@ def get_uri_scheme(uri: str) -> str:
         # So we add special handling here for schemes that are a single character
         scheme = "file"
     return scheme
+
+
+def get_uri_location(uri: str) -> str:
+    """
+    Get the location of a URI. If the parameter is not a url, assumes it is just a path
+
+    Parameters
+    ----------
+    uri : str
+        The URI to parse.
+
+    Returns
+    -------
+    str: Location part of the URL, without scheme
+    """
+    parsed = urlparse(uri)
+    if not parsed.netloc:
+        return parsed.path
+    else:
+        return parsed.netloc + parsed.path

python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lancedb"
-version = "0.1.4"
+version = "0.1.5"
 dependencies = ["pylance>=0.4.17", "ratelimiter", "retry", "tqdm"]
 description = "lancedb"
 authors = [

python/tests/test_db.py

@@ -97,3 +97,26 @@ def test_create_mode(tmp_path):
     )
     tbl = db.create_table("test", data=new_data, mode="overwrite")
     assert tbl.to_pandas().item.tolist() == ["fizz", "buzz"]
+
+
+def test_delete_table(tmp_path):
+    db = lancedb.connect(tmp_path)
+    data = pd.DataFrame(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", "bar"],
+            "price": [10.0, 20.0],
+        }
+    )
+    db.create_table("test", data=data)
+
+    with pytest.raises(Exception):
+        db.create_table("test", data=data)
+
+    assert db.table_names() == ["test"]
+
+    db.drop_table("test")
+    assert db.table_names() == []
+
+    db.create_table("test", data=data)
+    assert db.table_names() == ["test"]

python/tests/test_io.py

@@ -0,0 +1,49 @@
+#  Copyright 2023 LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import pytest
+
+import lancedb
+
+# You need to setup AWS credentials an a base path to run this test. Example
+#    AWS_PROFILE=default TEST_S3_BASE_URL=s3://my_bucket/dataset pytest tests/test_io.py
+
+@pytest.mark.skipif(
+    (os.environ.get("TEST_S3_BASE_URL") is None),
+    reason="please setup s3 base url",
+)
+def test_s3_io():
+    db = lancedb.connect(os.environ.get("TEST_S3_BASE_URL"))
+    assert db.table_names() == []
+
+    table = db.create_table(
+        "test",
+        data=[
+            {"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+            {"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
+        ],
+    )
+    rs = table.search([100, 100]).limit(1).to_df()
+    assert len(rs) == 1
+    assert rs["item"].iloc[0] == "bar"
+
+    rs = table.search([100, 100]).where("price < 15").limit(2).to_df()
+    assert len(rs) == 1
+    assert rs["item"].iloc[0] == "foo"
+
+    assert db.table_names() == ["test"]
+    assert "test" in db
+    assert len(db) == 1
+
+    assert db.open_table("test").name == db["test"].name

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.2"
+version = "0.1.0"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.1.2"
+version = "0.0.1"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"