From c385c55629b0806ff48a5b575df5c392917324fd Mon Sep 17 00:00:00 2001 From: gsilvestrin Date: Wed, 12 Jul 2023 16:52:04 -0700 Subject: [PATCH] feat(node): pull node binaries into separate packages (3) (#285) --- .github/workflows/node.yml | 12 ++- .github/workflows/npm-publish.yml | 137 ++++++++++++++++++++++++++++++ .gitignore | 2 + ci/build_linux_artifacts.sh | 72 ++++++++++++++++ ci/build_macos_artifacts.sh | 33 +++++++ node/.npmignore | 4 + node/README.md | 28 +++++- node/native.js | 37 ++++---- node/package-lock.json | 45 +++++++++- node/package.json | 30 ++++++- 10 files changed, 373 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/npm-publish.yml create mode 100644 ci/build_linux_artifacts.sh create mode 100644 ci/build_macos_artifacts.sh create mode 100644 node/.npmignore diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index b1a85421..076750e2 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -67,8 +67,12 @@ jobs: - name: Build run: | npm ci - npm run build npm run tsc + npm run build + npm run pack-build + npm install --no-save ./dist/vectordb-*.tgz + # Remove index.node to test with dependency installed + rm index.node - name: Test run: npm run test macos: @@ -94,8 +98,12 @@ jobs: - name: Build run: | npm ci - npm run build npm run tsc + npm run build + npm run pack-build + npm install --no-save ./dist/vectordb-*.tgz + # Remove index.node to test with dependency installed + rm index.node - name: Test run: | npm run test diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml new file mode 100644 index 00000000..cdc7b335 --- /dev/null +++ b/.github/workflows/npm-publish.yml @@ -0,0 +1,137 @@ +name: NPM Publish + +on: + release: + types: [ published ] + +jobs: + node: + runs-on: ubuntu-latest + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + defaults: + run: + shell: bash + working-directory: node + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 20 + cache: 'npm' + cache-dependency-path: node/package-lock.json + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Build + run: | + npm ci + npm run tsc + npm pack + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v3 + with: + name: node-package + path: | + node/vectordb-*.tgz + + node-macos: + runs-on: macos-12 + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + target: [x86_64-apple-darwin, aarch64-apple-darwin] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install system dependencies + run: brew install protobuf + - name: Install npm dependencies + run: | + cd node + npm ci + - name: Install rustup target + if: ${{ matrix.target == 'aarch64-apple-darwin' }} + run: rustup target add aarch64-apple-darwin + - name: Build MacOS native node modules + run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} + - name: Upload Darwin Artifacts + uses: actions/upload-artifact@v3 + with: + name: darwin-native + path: | + node/dist/vectordb-darwin*.tgz + + node-linux: + name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }}) + runs-on: ubuntu-latest + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + libc: + - gnu + # TODO: re-enable musl once we have refactored to pre-built containers + # Right now we have to build node from source which is too expensive. + # - musl + arch: + - x86_64 + # Building on aarch64 is too slow for now + # - aarch64 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Change owner to root (for npm) + # The docker container is run as root, so we need the files to be owned by root + # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773 + run: sudo chown -R root:root . + - name: Set up QEMU + if: ${{ matrix.arch == 'aarch64' }} + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Build Linux GNU native node modules + if: ${{ matrix.libc == 'gnu' }} + run: | + docker run \ + -v $(pwd):/io -w /io \ + rust:1.70-bookworm \ + bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu + - name: Build musl Linux native node modules + if: ${{ matrix.libc == 'musl' }} + run: | + docker run --platform linux/arm64/v8 \ + -v $(pwd):/io -w /io \ + quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \ + bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v3 + with: + name: linux-native + path: | + node/dist/vectordb-linux*.tgz + + release: + needs: [node, node-macos, node-linux] + runs-on: ubuntu-latest + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v3 + - name: Display structure of downloaded files + run: ls -R + - uses: actions/setup-node@v3 + with: + node-version: 20 + - name: Publish to NPM + env: + NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} + run: | + for filename in */*.tgz; do + npm publish $filename + done diff --git a/.gitignore b/.gitignore index 92421b13..1e455390 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ .DS_Store venv +.vscode + rust/target rust/Cargo.lock diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh new file mode 100644 index 00000000..2a9f6e5f --- /dev/null +++ b/ci/build_linux_artifacts.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Builds the Linux artifacts (node binaries). +# Usage: ./build_linux_artifacts.sh [target] +# Targets supported: +# - x86_64-unknown-linux-gnu:centos +# - aarch64-unknown-linux-gnu:centos +# - aarch64-unknown-linux-musl +# - x86_64-unknown-linux-musl + +# TODO: refactor this into a Docker container we can pull + +set -e + +setup_dependencies() { + echo "Installing system dependencies..." + if [[ $1 == *musl ]]; then + # musllinux + apk add openssl-dev + else + # rust / debian + apt update + apt install -y libssl-dev protobuf-compiler + fi +} + +install_node() { + echo "Installing node..." + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash + source "$HOME"/.bashrc + + if [[ $1 == *musl ]]; then + # This node version is 15, we need 16 or higher: + # apk add nodejs-current npm + # So instead we install from source (nvm doesn't provide binaries for musl): + nvm install -s --no-progress 17 + else + nvm install --no-progress 17 # latest that supports glibc 2.17 + fi +} + +build_node_binary() { + echo "Building node library for $1..." + pushd node + + npm ci + + if [[ $1 == *musl ]]; then + # This is needed for cargo to allow build cdylibs with musl + export RUSTFLAGS="-C target-feature=-crt-static" + fi + + # Cargo can run out of memory while pulling dependencies, especially when running + # in QEMU. This is a workaround for that. + export CARGO_NET_GIT_FETCH_WITH_CLI=true + + # We don't pass in target, since the native target here already matches + # We need to pass OPENSSL_LIB_DIR and OPENSSL_INCLUDE_DIR for static build to work https://github.com/sfackler/rust-openssl/issues/877 + OPENSSL_STATIC=1 OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu OPENSSL_INCLUDE_DIR=/usr/include/openssl/ npm run build-release + npm run pack-build + + popd +} + +TARGET=${1:-x86_64-unknown-linux-gnu} +# Others: +# aarch64-unknown-linux-gnu +# x86_64-unknown-linux-musl +# aarch64-unknown-linux-musl + +setup_dependencies $TARGET +install_node $TARGET +build_node_binary $TARGET diff --git a/ci/build_macos_artifacts.sh b/ci/build_macos_artifacts.sh new file mode 100644 index 00000000..15005767 --- /dev/null +++ b/ci/build_macos_artifacts.sh @@ -0,0 +1,33 @@ +# Builds the macOS artifacts (node binaries). +# Usage: ./ci/build_macos_artifacts.sh [target] +# Targets supported: x86_64-apple-darwin aarch64-apple-darwin + +prebuild_rust() { + # Building here for the sake of easier debugging. + pushd rust/ffi/node + echo "Building rust library for $1" + export RUST_BACKTRACE=1 + cargo build --release --target $1 + popd +} + +build_node_binaries() { + pushd node + echo "Building node library for $1" + npm run build-release -- --target $1 + npm run pack-build -- --target $1 + popd +} + +if [ -n "$1" ]; then + targets=$1 +else + targets="x86_64-apple-darwin aarch64-apple-darwin" +fi + +echo "Building artifacts for targets: $targets" +for target in $targets + do + prebuild_rust $target + build_node_binaries $target +done \ No newline at end of file diff --git a/node/.npmignore b/node/.npmignore new file mode 100644 index 00000000..3da85495 --- /dev/null +++ b/node/.npmignore @@ -0,0 +1,4 @@ +gen_test_data.py +index.node +dist/lancedb*.tgz +vectordb*.tgz \ No newline at end of file diff --git a/node/README.md b/node/README.md index b9fd75c3..b5021965 100644 --- a/node/README.md +++ b/node/README.md @@ -8,6 +8,10 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb) npm install vectordb ``` +This will download the appropriate native library for your platform. We currently +support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not +yet support Windows or musl-based Linux (such as Alpine Linux). + ## Usage ### Basic Example @@ -26,12 +30,34 @@ The [examples](./examples) folder contains complete examples. ## Development -Run the tests with +To build everything fresh: + +```bash +npm install +npm run tsc +npm run build +``` + +Then you should be able to run the tests with: ```bash npm test ``` +### Rebuilding Rust library + +```bash +npm run build +``` + +### Rebuilding Typescript + +```bash +npm run tsc +``` + +### Fix lints + To run the linter and have it automatically fix all errors ```bash diff --git a/node/native.js b/node/native.js index 0d4172a0..68aacade 100644 --- a/node/native.js +++ b/node/native.js @@ -12,29 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. +const { currentTarget } = require('@neon-rs/load'); + let nativeLib; -function getPlatformLibrary() { - if (process.platform === "darwin" && process.arch == "arm64") { - return require('./aarch64-apple-darwin.node'); - } else if (process.platform === "darwin" && process.arch == "x64") { - return require('./x86_64-apple-darwin.node'); - } else if (process.platform === "linux" && process.arch == "x64") { - return require('./x86_64-unknown-linux-gnu.node'); - } else { - throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`) - } -} - try { - nativeLib = require('./index.node') + nativeLib = require(`vectordb-${currentTarget()}`); } catch (e) { - if (e.code === "MODULE_NOT_FOUND") { - nativeLib = getPlatformLibrary(); - } else { - throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues'); - } + try { + // Might be developing locally, so try that. But don't expose that error + // to the user. + nativeLib = require("./index.node"); + } catch { + throw new Error(`vectordb: failed to load native library. + You may need to run \`npm install vectordb-${currentTarget()}\`. + + If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues + + Source error: ${e}`); + } } -module.exports = nativeLib - +// Dynamic require for runtime. +module.exports = nativeLib; diff --git a/node/package-lock.json b/node/package-lock.json index 8a58eccc..cbc89824 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -1,18 +1,28 @@ { "name": "vectordb", - "version": "0.1.10", + "version": "0.1.12", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "vectordb", - "version": "0.1.10", + "version": "0.1.12", + "cpu": [ + "x64", + "arm64" + ], "license": "Apache-2.0", + "os": [ + "darwin", + "linux" + ], "dependencies": { "@apache-arrow/ts": "^12.0.0", + "@neon-rs/load": "^0.0.74", "apache-arrow": "^12.0.0" }, "devDependencies": { + "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/chai-as-promised": "^7.1.5", "@types/mocha": "^10.0.1", @@ -37,6 +47,12 @@ "typedoc": "^0.24.7", "typedoc-plugin-markdown": "^3.15.3", "typescript": "*" + }, + "optionalDependencies": { + "vectordb-darwin-arm64": "0.1.12", + "vectordb-darwin-x64": "0.1.12", + "vectordb-linux-arm64-gnu": "0.1.12", + "vectordb-linux-x64-gnu": "0.1.12" } }, "node_modules/@apache-arrow/ts": { @@ -204,6 +220,20 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@neon-rs/cli": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", + "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", + "dev": true, + "bin": { + "neon": "index.js" + } + }, + "node_modules/@neon-rs/load": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", + "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4601,6 +4631,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@neon-rs/cli": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", + "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", + "dev": true + }, + "@neon-rs/load": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", + "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/node/package.json b/node/package.json index c679c3cf..5d6fa277 100644 --- a/node/package.json +++ b/node/package.json @@ -6,11 +6,13 @@ "types": "dist/index.d.ts", "scripts": { "tsc": "tsc -b", - "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics", + "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json", "build-release": "npm run build -- --release", "test": "npm run tsc; mocha -recursive dist/test", "lint": "eslint src --ext .js,.ts", - "clean": "rm -rf node_modules *.node dist/" + "clean": "rm -rf node_modules *.node dist/", + "pack-build": "neon pack-build", + "check-npm": "printenv && which node && which npm && npm --version" }, "repository": { "type": "git", @@ -25,6 +27,7 @@ "author": "Lance Devs", "license": "Apache-2.0", "devDependencies": { + "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/chai-as-promised": "^7.1.5", "@types/mocha": "^10.0.1", @@ -52,6 +55,29 @@ }, "dependencies": { "@apache-arrow/ts": "^12.0.0", + "@neon-rs/load": "^0.0.74", "apache-arrow": "^12.0.0" + }, + "os": [ + "darwin", + "linux" + ], + "cpu": [ + "x64", + "arm64" + ], + "neon": { + "targets": { + "x86_64-apple-darwin": "vectordb-darwin-x64", + "aarch64-apple-darwin": "vectordb-darwin-arm64", + "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu", + "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu" + } + }, + "optionalDependencies": { + "vectordb-darwin-arm64": "0.1.12", + "vectordb-darwin-x64": "0.1.12", + "vectordb-linux-x64-gnu": "0.1.12", + "vectordb-linux-arm64-gnu": "0.1.12" } }