From 8708fde3ef259c7e7d6afce37daba5df62ba2958 Mon Sep 17 00:00:00 2001 From: gsilvestrin Date: Fri, 16 Jun 2023 18:15:49 -0700 Subject: [PATCH] =?UTF-8?q?Revert=20"feat(node):=20pull=20node=20binaries?= =?UTF-8?q?=20into=20separate=20packages=20(2)=20(#1=E2=80=A6=20(#206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …97)" This reverts commit 0724d41c4b82bbafbc0ad159db64b3b44930942e. --- .github/workflows/node.yml | 12 +-- .github/workflows/npm-publish.yml | 137 ------------------------------ .gitignore | 2 - Cargo.lock | 4 +- ci/build_linux_artifacts.sh | 91 -------------------- ci/build_macos_artifacts.sh | 33 ------- node/.npmignore | 4 - node/README.md | 24 +----- node/native.js | 41 ++++----- node/package-lock.json | 45 +--------- node/package.json | 31 +------ 11 files changed, 32 insertions(+), 392 deletions(-) delete mode 100644 .github/workflows/npm-publish.yml delete mode 100644 ci/build_linux_artifacts.sh delete mode 100644 ci/build_macos_artifacts.sh delete mode 100644 node/.npmignore diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 076750e2..b1a85421 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -67,12 +67,8 @@ jobs: - name: Build run: | npm ci - npm run tsc npm run build - npm run pack-build - npm install --no-save ./dist/vectordb-*.tgz - # Remove index.node to test with dependency installed - rm index.node + npm run tsc - name: Test run: npm run test macos: @@ -98,12 +94,8 @@ jobs: - name: Build run: | npm ci - npm run tsc npm run build - npm run pack-build - npm install --no-save ./dist/vectordb-*.tgz - # Remove index.node to test with dependency installed - rm index.node + npm run tsc - name: Test run: | npm run test diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml deleted file mode 100644 index 6c55e0e7..00000000 --- a/.github/workflows/npm-publish.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: NPM Publish - -on: - release: - types: [ published ] - -jobs: - node: - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - defaults: - run: - shell: bash - working-directory: node - steps: - - name: Checkout - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: 'npm' - cache-dependency-path: node/package-lock.json - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Build - run: | - npm ci - npm run tsc - npm pack - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v3 - with: - name: node-package - path: | - node/vectordb-*.tgz - - node-macos: - runs-on: macos-12 - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - target: [x86_64-apple-darwin, aarch64-apple-darwin] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install system dependencies - run: brew install protobuf - - name: Install npm dependencies - run: | - cd node - npm ci - - name: Install rustup target - if: ${{ matrix.target == 'aarch64-apple-darwin' }} - run: rustup target add aarch64-apple-darwin - - name: Build MacOS native node modules - run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} - - name: Upload Darwin Artifacts - uses: actions/upload-artifact@v3 - with: - name: darwin-native - path: | - node/dist/vectordb-darwin*.tgz - - node-linux: - name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }}) - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - strategy: - fail-fast: false - matrix: - libc: - - gnu - # TODO: re-enable musl once we have refactored to pre-built containers - # Right now we have to build node from source which is too expensive. - # - musl - arch: - - x86_64 - # Building on aarch64 is too slow for now - # - aarch64 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Change owner to root (for npm) - # The docker container is run as root, so we need the files to be owned by root - # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773 - run: sudo chown -R root:root . - - name: Set up QEMU - if: ${{ matrix.arch == 'aarch64' }} - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Build Linux GNU native node modules - if: ${{ matrix.libc == 'gnu' }} - run: | - docker run \ - -v $(pwd):/io -w /io \ - quay.io/pypa/manylinux2014_${{ matrix.arch }} \ - bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu - - name: Build musl Linux native node modules - if: ${{ matrix.libc == 'musl' }} - run: | - docker run --platform linux/arm64/v8 \ - -v $(pwd):/io -w /io \ - quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \ - bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl - - name: Upload Linux Artifacts - uses: actions/upload-artifact@v3 - with: - name: linux-native - path: | - node/dist/vectordb-linux*.tgz - - release: - needs: [node, node-macos, node-linux] - runs-on: ubuntu-latest - # Only runs on tags that matches the make-release action - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/download-artifact@v3 - - name: Display structure of downloaded files - run: ls -R - - uses: actions/setup-node@v3 - with: - node-version: 20 - - name: Publish to NPM - env: - NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} - run: | - for filename in */*.tgz; do - npm publish $filename - done diff --git a/.gitignore b/.gitignore index 7855bca1..dfee6bdf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ **/__pycache__ .DS_Store -.vscode - rust/target rust/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 3f1d7e54..4f044d52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3386,7 +3386,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vectordb" -version = "0.1.6" +version = "0.0.1" dependencies = [ "arrow-array", "arrow-data", @@ -3401,7 +3401,7 @@ dependencies = [ [[package]] name = "vectordb-node" -version = "0.1.6" +version = "0.1.0" dependencies = [ "arrow-array", "arrow-ipc", diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh deleted file mode 100644 index 8ed95a2e..00000000 --- a/ci/build_linux_artifacts.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# Builds the Linux artifacts (node binaries). -# Usage: ./build_linux_artifacts.sh [target] -# Targets supported: -# - x86_64-unknown-linux-gnu:centos -# - aarch64-unknown-linux-gnu:centos -# - aarch64-unknown-linux-musl -# - x86_64-unknown-linux-musl - -# TODO: refactor this into a Docker container we can pull - -set -e - -setup_dependencies() { - echo "Installing system dependencies..." - if [[ $1 == *musl ]]; then - # musllinux - apk add openssl-dev - else - # manylinux2014 - yum install -y openssl-devel unzip - fi - - if [[ $1 == x86_64* ]]; then - ARCH=x86_64 - else - # gnu target - ARCH=aarch_64 - fi - - # Install new enough protobuf (yum-provided is old) - PB_REL=https://github.com/protocolbuffers/protobuf/releases - PB_VERSION=23.1 - curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip - unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local -} - -install_node() { - echo "Installing node..." - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash - source "$HOME"/.bashrc - - if [[ $1 == *musl ]]; then - # This node version is 15, we need 16 or higher: - # apk add nodejs-current npm - # So instead we install from source (nvm doesn't provide binaries for musl): - nvm install -s --no-progress 17 - else - nvm install --no-progress 17 # latest that supports glibc 2.17 - fi -} - -install_rust() { - echo "Installing rust..." - curl https://sh.rustup.rs -sSf | bash -s -- -y - export PATH="$PATH:/root/.cargo/bin" -} - -build_node_binary() { - echo "Building node library for $1..." - pushd node - - npm ci - - if [[ $1 == *musl ]]; then - # This is needed for cargo to allow build cdylibs with musl - export RUSTFLAGS="-C target-feature=-crt-static" - fi - - # Cargo can run out of memory while pulling dependencies, espcially when running - # in QEMU. This is a workaround for that. - export CARGO_NET_GIT_FETCH_WITH_CLI=true - - # We don't pass in target, since the native target here already matches - # and openblas-src doesn't do well with cross-compilation. - npm run build-release - npm run pack-build - - popd -} - -TARGET=${1:-x86_64-unknown-linux-gnu} -# Others: -# aarch64-unknown-linux-gnu -# x86_64-unknown-linux-musl -# aarch64-unknown-linux-musl - -setup_dependencies $TARGET -install_node $TARGET -install_rust -build_node_binary $TARGET diff --git a/ci/build_macos_artifacts.sh b/ci/build_macos_artifacts.sh deleted file mode 100644 index 15005767..00000000 --- a/ci/build_macos_artifacts.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Builds the macOS artifacts (node binaries). -# Usage: ./ci/build_macos_artifacts.sh [target] -# Targets supported: x86_64-apple-darwin aarch64-apple-darwin - -prebuild_rust() { - # Building here for the sake of easier debugging. - pushd rust/ffi/node - echo "Building rust library for $1" - export RUST_BACKTRACE=1 - cargo build --release --target $1 - popd -} - -build_node_binaries() { - pushd node - echo "Building node library for $1" - npm run build-release -- --target $1 - npm run pack-build -- --target $1 - popd -} - -if [ -n "$1" ]; then - targets=$1 -else - targets="x86_64-apple-darwin aarch64-apple-darwin" -fi - -echo "Building artifacts for targets: $targets" -for target in $targets - do - prebuild_rust $target - build_node_binaries $target -done \ No newline at end of file diff --git a/node/.npmignore b/node/.npmignore deleted file mode 100644 index 3da85495..00000000 --- a/node/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -gen_test_data.py -index.node -dist/lancedb*.tgz -vectordb*.tgz \ No newline at end of file diff --git a/node/README.md b/node/README.md index 01ebbce4..9302cf5b 100644 --- a/node/README.md +++ b/node/README.md @@ -8,10 +8,6 @@ A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb) npm install vectordb ``` -This will download the appropriate native library for your platform. We currently -support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not -yet support Windows or musl-based Linux (such as Alpine Linux). - ## Usage ### Basic Example @@ -28,34 +24,18 @@ The [examples](./examples) folder contains complete examples. ## Development -To build everything fresh: +The LanceDB javascript is built with npm: ```bash -npm install npm run tsc -npm run build ``` -Then you should be able to run the tests with: +Run the tests with ```bash npm test ``` -### Rebuilding Rust library - -```bash -npm run build -``` - -### Rebuilding Typescript - -```bash -npm run tsc -``` - -### Fix lints - To run the linter and have it automatically fix all errors ```bash diff --git a/node/native.js b/node/native.js index 68aacade..0d4172a0 100644 --- a/node/native.js +++ b/node/native.js @@ -12,26 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { currentTarget } = require('@neon-rs/load'); - let nativeLib; -try { - nativeLib = require(`vectordb-${currentTarget()}`); -} catch (e) { - try { - // Might be developing locally, so try that. But don't expose that error - // to the user. - nativeLib = require("./index.node"); - } catch { - throw new Error(`vectordb: failed to load native library. - You may need to run \`npm install vectordb-${currentTarget()}\`. - - If that does not work, please file a bug report at https://github.com/lancedb/lancedb/issues - - Source error: ${e}`); - } +function getPlatformLibrary() { + if (process.platform === "darwin" && process.arch == "arm64") { + return require('./aarch64-apple-darwin.node'); + } else if (process.platform === "darwin" && process.arch == "x64") { + return require('./x86_64-apple-darwin.node'); + } else if (process.platform === "linux" && process.arch == "x64") { + return require('./x86_64-unknown-linux-gnu.node'); + } else { + throw new Error(`vectordb: unsupported platform ${process.platform}_${process.arch}. Please file a bug report at https://github.com/lancedb/lancedb/issues`) + } } -// Dynamic require for runtime. -module.exports = nativeLib; +try { + nativeLib = require('./index.node') +} catch (e) { + if (e.code === "MODULE_NOT_FOUND") { + nativeLib = getPlatformLibrary(); + } else { + throw new Error('vectordb: failed to load native library. Please file a bug report at https://github.com/lancedb/lancedb/issues'); + } +} + +module.exports = nativeLib + diff --git a/node/package-lock.json b/node/package-lock.json index 5a9d4826..e0179ab0 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -1,28 +1,18 @@ { "name": "vectordb", - "version": "0.1.5", + "version": "0.1.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "vectordb", - "version": "0.1.5", - "cpu": [ - "x64", - "arm64" - ], + "version": "0.1.8", "license": "Apache-2.0", - "os": [ - "darwin", - "linux" - ], "dependencies": { "@apache-arrow/ts": "^12.0.0", - "@neon-rs/load": "^0.0.74", "apache-arrow": "^12.0.0" }, "devDependencies": { - "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", @@ -45,12 +35,6 @@ "typedoc": "^0.24.7", "typedoc-plugin-markdown": "^3.15.3", "typescript": "*" - }, - "optionalDependencies": { - "vectordb-darwin-arm64": "0.1.2", - "vectordb-darwin-x64": "0.1.2", - "vectordb-linux-arm64-gnu": "0.1.2", - "vectordb-linux-x64-gnu": "0.1.2" } }, "node_modules/@apache-arrow/ts": { @@ -218,20 +202,6 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "node_modules/@neon-rs/cli": { - "version": "0.0.74", - "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", - "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", - "dev": true, - "bin": { - "neon": "index.js" - } - }, - "node_modules/@neon-rs/load": { - "version": "0.0.74", - "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", - "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4631,17 +4601,6 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, - "@neon-rs/cli": { - "version": "0.0.74", - "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.74.tgz", - "integrity": "sha512-9lPmNmjej5iKKOTMPryOMubwkgMRyTWRuaq1yokASvI5mPhr2kzPN7UVjdCOjQvpunNPngR9yAHoirpjiWhUHw==", - "dev": true - }, - "@neon-rs/load": { - "version": "0.0.74", - "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", - "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==" - }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/node/package.json b/node/package.json index b3eceaab..39c24e31 100644 --- a/node/package.json +++ b/node/package.json @@ -6,14 +6,11 @@ "types": "dist/index.d.ts", "scripts": { "tsc": "tsc -b", - "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json", + "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics", "build-release": "npm run build -- --release", - "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node", "test": "mocha -recursive dist/test", "lint": "eslint src --ext .js,.ts", - "clean": "rm -rf node_modules *.node dist/", - "pack-build": "neon pack-build", - "check-npm": "printenv && which node && which npm && npm --version" + "clean": "rm -rf node_modules *.node dist/" }, "repository": { "type": "git", @@ -28,7 +25,6 @@ "author": "Lance Devs", "license": "Apache-2.0", "devDependencies": { - "@neon-rs/cli": "^0.0.74", "@types/chai": "^4.3.4", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", @@ -54,29 +50,6 @@ }, "dependencies": { "@apache-arrow/ts": "^12.0.0", - "@neon-rs/load": "^0.0.74", "apache-arrow": "^12.0.0" - }, - "os": [ - "darwin", - "linux" - ], - "cpu": [ - "x64", - "arm64" - ], - "neon": { - "targets": { - "x86_64-apple-darwin": "vectordb-darwin-x64", - "aarch64-apple-darwin": "vectordb-darwin-arm64", - "x86_64-unknown-linux-gnu": "vectordb-linux-x64-gnu", - "aarch64-unknown-linux-gnu": "vectordb-linux-arm64-gnu" - } - }, - "optionalDependencies": { - "vectordb-darwin-arm64": "0.1.8", - "vectordb-darwin-x64": "0.1.8", - "vectordb-linux-x64-gnu": "0.1.8", - "vectordb-linux-arm64-gnu": "0.1.8" } }