fixes for action

.github/workflows/node.yml

@@ -67,8 +67,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: npm run test
   macos:
@@ -94,8 +96,10 @@ jobs:
     - name: Build
       run: |
         npm ci
-        npm run build
         npm run tsc
+        npm run build
+        npm run pack-build
+        npm install --no-save ./dist/vectordb-*.tgz
     - name: Test
       run: |
         npm run test

.github/workflows/release.yml

@@ -0,0 +1,181 @@
+name: Prepare Release
+
+# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+
+on:
+  push:
+    tags:
+      - v*
+
+jobs:
+  draft-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          prerelease: true # hardcoded on for now
+          generate_release_notes: true
+  
+  rust:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust/vectordb
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Package Rust
+        run: cargo package --all-features
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: target/vectordb-*.crate
+    
+  python:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel
+        run: |
+          pip install wheel
+          python setup.py sdist bdist_wheel
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: |
+            python/dist/lancedb-*.tar.gz 
+            python/dist/lancedb-*.whl
+
+  node:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: node/package-lock.json
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build
+        run: |
+          npm ci
+          npm run tsc
+          npm pack
+      - uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: node/vectordb-*.tgz
+
+  node-macos:
+    runs-on: macos-12
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install system dependencies
+      run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Build MacOS native node modules
+      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-darwin*.tgz
+
+  node-linux:
+    runs-on: ubuntu-latest
+    needs: draft-release
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
+          - aarch64-unknown-linux-gnu
+          - aarch64-unknown-linux-musl
+          - x86_64-unknown-linux-musl
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Setup Rust
+      uses: ATiltedTree/setup-rust@v1
+      with:
+        rust-version: stable
+    - name: Install system dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler libssl-dev
+        cargo install cross
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
+    - name: Build Linux native node modules
+      run: bash ci/build_linux_artifacts.sh ${{ matrix.target }}
+    - uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: node/dist/vectordb-linux*.tgz
+
+  release:
+    needs: [python, node, node-macos, node-linux, rust]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python -m twine upload --non-interactive \
+            --skip-existing \
+            --repository testpypi python/dist/*
+      - name: Publish to NPM
+        run: |
+          for filename in node/dist/*.tgz; do
+            npm publish --dry-run $filename
+          done
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          cargo publish --dry-run --no-verify rust/target/vectordb-*.crate
+      # - uses: softprops/action-gh-release@v1
+      #   with:
+      #     draft: false

.gitignore

@@ -4,6 +4,8 @@
 **/__pycache__
 .DS_Store
 
+.vscode
+
 rust/target
 rust/Cargo.lock
 

Cargo.lock

@@ -3356,7 +3356,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -3367,7 +3367,7 @@ dependencies = [
 
 [[package]]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

Cross.toml

@@ -0,0 +1,14 @@
+# These make sure our builds are compatible with old glibc versions.
+[target.x86_64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
+
+[target.aarch64-unknown-linux-gnu]
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+]
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"

ci/build_linux_artifacts.sh

@@ -0,0 +1,36 @@
+# Builds the Linux artifacts (node binaries).
+# Usage: ./build_linux_artifacts.sh [target]
+# Targets supported: 
+# - x86_64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-gnu:centos
+# - aarch64-unknown-linux-musl
+# - x86_64-unknown-linux-musl
+
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
+
+set -e
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+        export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+    targets="aarch64-unknown-linux-gnu"
+fi
+build_node_binaries $targets

ci/build_macos_artifacts.sh

@@ -0,0 +1,22 @@
+# Builds the macOS artifacts (node binaries).
+# Usage: ./build_macos_artifacts.sh [target]
+# Targets supported: x86_64-apple-darwin aarch64-apple-darwin
+
+build_node_binaries() {
+    pushd node
+    
+    for target in $1
+    do
+        echo "Building node library for $target"
+        npm run build-release -- --target $target
+        npm run pack-build -- --target $target
+    done
+    popd
+}
+
+if [ -n "$1" ]; then
+    targets=$1
+else
+    targets="x86_64-apple-darwin aarch64-apple-darwin"
+fi
+build_node_binaries $targets

ci/ubuntu_build.dockerfile

@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1

node/.npmignore

@@ -0,0 +1,2 @@
+gen_test_data.py
+index.node

node/README.md

@@ -8,6 +8,9 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb)
 npm install vectordb
 ```
 
+This will download the appropriate native library for your platform. We currently
+support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
+
 ## Usage
 
 ### Basic Example
@@ -24,6 +27,19 @@ The [examples](./examples) folder contains complete examples.
 
 ## Development
 
+Build and install the rust library with:
+
+```bash
+npm run build
+npm run pack-build
+npm install --no-save ./dist/vectordb-*.tgz
+```
+
+`npm run build` builds the Rust library, `npm run pack-build` packages the Rust
+binary into an npm module called `@vectordb/<platform>` (for example, 
+`@vectordb/darwin-arm64.node`), and then `npm run install ...` installs that
+module.
+
 The LanceDB javascript is built with npm:
 
 ```bash

node/native.js

@@ -12,29 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+const { currentTarget } = require('@neon-rs/load');
+
 let nativeLib;
 
-function getPlatformLibrary() {
-    if (process.platform === "darwin" && process.arch == "arm64") {
-        return require('./aarch64-apple-darwin.node');
-    } else if (process.platform === "darwin" && process.arch == "x64") {
-        return require('./x86_64-apple-darwin.node');
-    } else if (process.platform === "linux" && process.arch == "x64") {
-        return require('./x86_64-unknown-linux-gnu.node');
-    } else {
-        throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`)
-    }
-}
-
 try {
-    nativeLib = require('./index.node')
+    nativeLib = require(`@vectordb/${currentTarget()}`);
 } catch (e) {
-    if (e.code === "MODULE_NOT_FOUND") {
-        nativeLib = getPlatformLibrary();
-    } else {
-        throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues');
-    }
+    throw new Error(`vectordb: failed to load native library.
+  You may need to run \`npm install @vectordb/${currentTarget()}\`.
+
+  If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues
+
+  Source error: ${e}`);
 }
 
-module.exports = nativeLib
-
+// Dynamic require for runtime.
+module.exports = nativeLib;

node/package-lock.json

@@ -7,12 +7,26 @@
     "": {
       "name": "vectordb",
       "version": "0.1.1",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
       "license": "Apache-2.0",
+      "os": [
+        "darwin",
+        "linux"
+      ],
       "dependencies": {
         "@apache-arrow/ts": "^12.0.0",
+        "@neon-rs/load": "^0.0.74",
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1",
         "apache-arrow": "^12.0.0"
       },
       "devDependencies": {
+        "@neon-rs/cli": "^0.0.74",
         "@types/chai": "^4.3.4",
         "@types/mocha": "^10.0.1",
         "@types/node": "^18.16.2",
@@ -30,6 +44,12 @@
         "ts-node": "^10.9.1",
         "ts-node-dev": "^2.0.0",
         "typescript": "*"
+      },
+      "optionalDependencies": {
+        "@vectordb/darwin-arm64": "0.1.1",
+        "@vectordb/darwin-x64": "0.1.1",
+        "@vectordb/linux-x64-gnu": "0.1.1",
+        "@vectordb/linux-x64-musl": "0.1.1"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -197,6 +217,20 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "node_modules/@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true,
+      "bin": {
+        "neon": "index.js"
+      }
+    },
+    "node_modules/@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -4191,6 +4225,17 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
+    "@neon-rs/cli": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz",
+      "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==",
+      "dev": true
+    },
+    "@neon-rs/load": {
+      "version": "0.0.74",
+      "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz",
+      "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg=="
+    },
     "@nodelib/fs.scandir": {
       "version": "2.1.5",
       "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",

node/package.json

@@ -1,15 +1,17 @@
 {
   "name": "vectordb",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
+    "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json",
     "build-release": "npm run build -- --release",
+    "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
-    "lint": "eslint src --ext .js,.ts"
+    "lint": "eslint src --ext .js,.ts",
+    "pack-build": "neon pack-build"
   },
   "repository": {
     "type": "git",
@@ -24,6 +26,7 @@
   "author": "Lance Devs",
   "license": "Apache-2.0",
   "devDependencies": {
+    "@neon-rs/cli": "^0.0.74",
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
     "@types/node": "^18.16.2",
@@ -44,6 +47,33 @@
   },
   "dependencies": {
     "@apache-arrow/ts": "^12.0.0",
+    "@neon-rs/load": "^0.0.74",
     "apache-arrow": "^12.0.0"
+  },
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
+  "neon": {
+    "targets": {
+      "x86_64-apple-darwin": "@vectordb/darwin-x64",
+      "aarch64-apple-darwin": "@vectordb/darwin-arm64",
+      "x86_64-unknown-linux-gnu": "@vectordb/linux-x64-gnu",
+      "x86_64-unknown-linux-musl": "@vectordb/linux-x64-musl",
+      "aarch64-unknown-linux-gnu": "@vectordb/linux-arm64-gnu",
+      "aarch64-unknown-linux-musl": "@vectordb/linux-arm64-musl"
+    }
+  },
+  "optionalDependencies": {
+    "@vectordb/darwin-arm64": "0.1.2",
+    "@vectordb/darwin-x64": "0.1.2",
+    "@vectordb/linux-x64-gnu": "0.1.2",
+    "@vectordb/linux-x64-musl": "0.1.2",
+    "@vectordb/linux-arm64-gnu": "0.1.2",
+    "@vectordb/linux-arm64-musl": "0.1.2"
   }
 }

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.1.0"
+version = "0.1.2"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/ffi/node/release_process.md

@@ -0,0 +1,85 @@
+
+How to release the node module
+
+### 1. Bump the versions
+
+<!-- TODO: we also need to bump the optional dependencies for node! -->
+
+```shell
+pushd rust/vectordb
+cargo bump minor
+popd
+
+pushd rust/ffi/node
+cargo bump minor
+popd
+
+pushd python
+cargo bump minor
+popd
+
+pushd node
+npm version minor
+popd
+
+git add -u
+git commit -m "Bump versions"
+git push
+```
+
+### 2. Push a new tag
+
+```shell
+git tag vX.X.X
+git push --tag vX.X.X
+```
+
+When the tag is pushed, GitHub actions will start building the libraries and
+will upload them to a draft release. Wait for those jobs to complete.
+
+### 3. Publish the release
+
+Once the jobs are complete, you can edit the 
+
+2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
+   building the native libraries and uploading them to a draft release. Wait for
+   those jobs to complete.
+3. If the libraries are successful, edit the changelog and then publish the
+   release. Once you publish, a new action will start and upload all the 
+   release artifacts to npm.
+
+## Manual process
+
+You can build the artifacts locally on a MacOS machine.
+
+### Build the MacOS release libraries
+
+One-time setup:
+
+```shell
+rustup target add x86_64-apple-darwin aarch64-apple-darwin
+```
+
+To build:
+
+```shell
+bash ci/build_macos_artifacts.sh
+```
+
+### Build the Linux release libraries
+
+One-time setup, building the Docker container
+
+```shell
+cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+```
+
+To build:
+
+```shell
+docker run \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v $(pwd):/io -w /io \
+    lancedb-node-build \
+    bash ci/build_linux_artifacts.sh
+```

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.0.1"
+version = "0.1.2"
 edition = "2021"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"