From 53d1535de1bdce77051fb9cfaec37ce424b5af84 Mon Sep 17 00:00:00 2001 From: "Mr. Doge" <42662615+FuPeiJiang@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:53:19 -0500 Subject: [PATCH] ci: musl x64,arm64 (#1853) untested 4 artifacts at: https://github.com/FuPeiJiang/lancedb/actions/runs/11926579058 node-native-linux-aarch64-musl 22.6 MB node-native-linux-x86_64-musl 23.6 MB nodejs-native-linux-aarch64-musl 26.7 MB nodejs-native-linux-x86_64-musl 27 MB this follows the same process as: https://github.com/lancedb/lancedb/pull/1816#issuecomment-2484816669 Closes #1388 Closes #1107 --------- Co-authored-by: Will Jones --- .bumpversion.toml | 10 ++ .cargo/config.toml | 3 + .github/workflows/npm-publish.yml | 120 +++++++++++++++++++++-- ci/manylinux_node/build_lancedb.sh | 3 +- ci/manylinux_node/build_vectordb.sh | 5 +- node/package.json | 6 +- nodejs/npm/linux-arm64-musl/README.md | 3 + nodejs/npm/linux-arm64-musl/package.json | 13 +++ nodejs/npm/linux-x64-musl/README.md | 3 + nodejs/npm/linux-x64-musl/package.json | 13 +++ nodejs/package.json | 6 +- 11 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 nodejs/npm/linux-arm64-musl/README.md create mode 100644 nodejs/npm/linux-arm64-musl/package.json create mode 100644 nodejs/npm/linux-x64-musl/README.md create mode 100644 nodejs/npm/linux-x64-musl/package.json diff --git a/.bumpversion.toml b/.bumpversion.toml index 31df55db..0a723951 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -87,6 +87,16 @@ glob = "node/package.json" replace = "\"@lancedb/vectordb-linux-x64-gnu\": \"{new_version}\"" search = "\"@lancedb/vectordb-linux-x64-gnu\": \"{current_version}\"" +[[tool.bumpversion.files]] +glob = "node/package.json" +replace = "\"@lancedb/vectordb-linux-arm64-musl\": \"{new_version}\"" +search = "\"@lancedb/vectordb-linux-arm64-musl\": \"{current_version}\"" + +[[tool.bumpversion.files]] +glob = "node/package.json" +replace = "\"@lancedb/vectordb-linux-x64-musl\": \"{new_version}\"" +search = "\"@lancedb/vectordb-linux-x64-musl\": \"{current_version}\"" + [[tool.bumpversion.files]] glob = "node/package.json" replace = "\"@lancedb/vectordb-win32-x64-msvc\": \"{new_version}\"" diff --git a/.cargo/config.toml b/.cargo/config.toml index 7a5e31a8..709609e9 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -31,6 +31,9 @@ rustflags = [ [target.x86_64-unknown-linux-gnu] rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=-crt-static,+avx2,+fma,+f16c"] + [target.aarch64-apple-darwin] rustflags = ["-C", "target-cpu=apple-m1", "-C", "target-feature=+neon,+fp16,+fhm,+dotprod"] diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 354a1722..c0ae7c20 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -101,7 +101,7 @@ jobs: path: | nodejs/dist/*.node - node-linux: + node-linux-gnu: name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu) runs-on: ${{ matrix.config.runner }} # Only runs on tags that matches the make-release action @@ -137,11 +137,63 @@ jobs: - name: Upload Linux Artifacts uses: actions/upload-artifact@v4 with: - name: node-native-linux-${{ matrix.config.arch }} + name: node-native-linux-${{ matrix.config.arch }}-gnu path: | node/dist/lancedb-vectordb-linux*.tgz - nodejs-linux: + node-linux-musl: + name: vectordb (${{ matrix.config.arch}}-unknown-linux-musl) + runs-on: ubuntu-latest + container: alpine:edge + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + config: + - arch: x86_64 + - arch: aarch64 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install common dependencies + run: | + apk add protobuf-dev curl clang mold grep npm bash + curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0 + echo "source $HOME/.cargo/env" >> saved_env + echo "export CC=clang" >> saved_env + echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env + - name: Configure aarch64 build + if: ${{ matrix.config.arch == 'aarch64' }} + run: | + source "$HOME/.cargo/env" + rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0 + crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained) + sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib + apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/ + curl -sSf $apk_url > apk_list + for pkg in gcc libgcc musl; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done + mkdir -p $sysroot_lib + echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so + cp usr/lib/libgcc_s.so.1 $sysroot_lib + cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib + cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so + echo '!' > $sysroot_lib/libdl.a + (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib) + echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env + echo "export RUSTFLAGS='-Ctarget-cpu=apple-m1 -Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env + - name: Build Linux Artifacts + run: | + source ./saved_env + bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v4 + with: + name: node-native-linux-${{ matrix.config.arch }}-musl + path: | + node/dist/lancedb-vectordb-linux*.tgz + + nodejs-linux-gnu: name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu runs-on: ${{ matrix.config.runner }} # Only runs on tags that matches the make-release action @@ -178,7 +230,7 @@ jobs: - name: Upload Linux Artifacts uses: actions/upload-artifact@v4 with: - name: nodejs-native-linux-${{ matrix.config.arch }} + name: nodejs-native-linux-${{ matrix.config.arch }}-gnu path: | nodejs/dist/*.node # The generic files are the same in all distros so we just pick @@ -192,6 +244,62 @@ jobs: nodejs/dist/* !nodejs/dist/*.node + nodejs-linux-musl: + name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl + runs-on: ubuntu-latest + container: alpine:edge + # Only runs on tags that matches the make-release action + if: startsWith(github.ref, 'refs/tags/v') + strategy: + fail-fast: false + matrix: + config: + - arch: x86_64 + - arch: aarch64 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install common dependencies + run: | + apk add protobuf-dev curl clang mold grep npm bash openssl-dev openssl-libs-static + curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y --default-toolchain 1.80.0 + echo "source $HOME/.cargo/env" >> saved_env + echo "export CC=clang" >> saved_env + echo "export RUSTFLAGS='-Ctarget-cpu=haswell -Ctarget-feature=-crt-static,+avx2,+fma,+f16c -Clinker=clang -Clink-arg=-fuse-ld=mold'" >> saved_env + echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=/usr/include" >> saved_env + echo "export X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib" >> saved_env + - name: Configure aarch64 build + if: ${{ matrix.config.arch == 'aarch64' }} + run: | + source "$HOME/.cargo/env" + rustup target add aarch64-unknown-linux-musl --toolchain 1.80.0 + crt=$(realpath $(dirname $(rustup which rustc))/../lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained) + sysroot_lib=/usr/aarch64-unknown-linux-musl/usr/lib + apk_url=https://dl-cdn.alpinelinux.org/alpine/latest-stable/main/aarch64/ + curl -sSf $apk_url > apk_list + for pkg in gcc libgcc musl openssl-dev openssl-libs-static; do curl -sSf $apk_url$(cat apk_list | grep -oP '(?<=")'$pkg'-\d.*?(?=")') | tar zxf -; done + mkdir -p $sysroot_lib + echo 'GROUP ( libgcc_s.so.1 -lgcc )' > $sysroot_lib/libgcc_s.so + cp usr/lib/libgcc_s.so.1 $sysroot_lib + cp usr/lib/gcc/aarch64-alpine-linux-musl/*/libgcc.a $sysroot_lib + cp lib/ld-musl-aarch64.so.1 $sysroot_lib/libc.so + echo '!' > $sysroot_lib/libdl.a + (cd $crt && cp crti.o crtbeginS.o crtendS.o crtn.o -t $sysroot_lib) + echo "export CARGO_BUILD_TARGET=aarch64-unknown-linux-musl" >> saved_env + echo "export RUSTFLAGS='-Ctarget-feature=-crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=--target=aarch64-unknown-linux-musl -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-musl -Clink-arg=-lc'" >> saved_env + echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_INCLUDE_DIR=$(realpath usr/include)" >> saved_env + echo "export AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=$(realpath usr/lib)" >> saved_env + - name: Build Linux Artifacts + run: | + source ./saved_env + bash ci/manylinux_node/build_lancedb.sh ${{ matrix.config.arch }} + - name: Upload Linux Artifacts + uses: actions/upload-artifact@v4 + with: + name: nodejs-native-linux-${{ matrix.config.arch }}-musl + path: | + nodejs/dist/*.node + node-windows: name: vectordb ${{ matrix.target }} runs-on: windows-2022 @@ -460,7 +568,7 @@ jobs: release: name: vectordb NPM Publish - needs: [node, node-macos, node-linux, node-windows] + needs: [node, node-macos, node-linux-gnu, node-linux-musl, node-windows] runs-on: ubuntu-latest # Only runs on tags that matches the make-release action if: startsWith(github.ref, 'refs/tags/v') @@ -500,7 +608,7 @@ jobs: release-nodejs: name: lancedb NPM Publish - needs: [nodejs-macos, nodejs-linux, nodejs-windows] + needs: [nodejs-macos, nodejs-linux-gnu, nodejs-linux-musl, nodejs-windows] runs-on: ubuntu-latest # Only runs on tags that matches the make-release action if: startsWith(github.ref, 'refs/tags/v') diff --git a/ci/manylinux_node/build_lancedb.sh b/ci/manylinux_node/build_lancedb.sh index b1fd60f8..597319c4 100644 --- a/ci/manylinux_node/build_lancedb.sh +++ b/ci/manylinux_node/build_lancedb.sh @@ -11,7 +11,8 @@ fi export OPENSSL_STATIC=1 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl -source $HOME/.bashrc +#Alpine doesn't have .bashrc +FILE=$HOME/.bashrc && test -f $FILE && source $FILE cd nodejs npm ci diff --git a/ci/manylinux_node/build_vectordb.sh b/ci/manylinux_node/build_vectordb.sh index 101719e6..36c11834 100755 --- a/ci/manylinux_node/build_vectordb.sh +++ b/ci/manylinux_node/build_vectordb.sh @@ -5,13 +5,14 @@ ARCH=${1:-x86_64} if [ "$ARCH" = "x86_64" ]; then export OPENSSL_LIB_DIR=/usr/local/lib64/ -else +else export OPENSSL_LIB_DIR=/usr/local/lib/ fi export OPENSSL_STATIC=1 export OPENSSL_INCLUDE_DIR=/usr/local/include/openssl -source $HOME/.bashrc +#Alpine doesn't have .bashrc +FILE=$HOME/.bashrc && test -f $FILE && source $FILE cd node npm ci diff --git a/node/package.json b/node/package.json index bd679d5a..9fab307f 100644 --- a/node/package.json +++ b/node/package.json @@ -89,10 +89,12 @@ } }, "optionalDependencies": { - "@lancedb/vectordb-darwin-arm64": "0.13.0", "@lancedb/vectordb-darwin-x64": "0.13.0", - "@lancedb/vectordb-linux-arm64-gnu": "0.13.0", + "@lancedb/vectordb-darwin-arm64": "0.13.0", "@lancedb/vectordb-linux-x64-gnu": "0.13.0", + "@lancedb/vectordb-linux-arm64-gnu": "0.13.0", + "@lancedb/vectordb-linux-x64-musl": "0.13.0", + "@lancedb/vectordb-linux-arm64-musl": "0.13.0", "@lancedb/vectordb-win32-x64-msvc": "0.13.0", "@lancedb/vectordb-win32-arm64-msvc": "0.13.0" } diff --git a/nodejs/npm/linux-arm64-musl/README.md b/nodejs/npm/linux-arm64-musl/README.md new file mode 100644 index 00000000..b0cffebc --- /dev/null +++ b/nodejs/npm/linux-arm64-musl/README.md @@ -0,0 +1,3 @@ +# `@lancedb/lancedb-linux-arm64-musl` + +This is the **aarch64-unknown-linux-musl** binary for `@lancedb/lancedb` \ No newline at end of file diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json new file mode 100644 index 00000000..b41c00bf --- /dev/null +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -0,0 +1,13 @@ +{ + "name": "@lancedb/lancedb-linux-arm64-musl", + "version": "0.13.0", + "os": ["linux"], + "cpu": ["arm64"], + "main": "lancedb.linux-arm64-musl.node", + "files": ["lancedb.linux-arm64-musl.node"], + "license": "Apache 2.0", + "engines": { + "node": ">= 18" + }, + "libc": ["musl"] +} \ No newline at end of file diff --git a/nodejs/npm/linux-x64-musl/README.md b/nodejs/npm/linux-x64-musl/README.md new file mode 100644 index 00000000..36d46b6b --- /dev/null +++ b/nodejs/npm/linux-x64-musl/README.md @@ -0,0 +1,3 @@ +# `@lancedb/lancedb-linux-x64-musl` + +This is the **x86_64-unknown-linux-musl** binary for `@lancedb/lancedb` \ No newline at end of file diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json new file mode 100644 index 00000000..6b42c04c --- /dev/null +++ b/nodejs/npm/linux-x64-musl/package.json @@ -0,0 +1,13 @@ +{ + "name": "@lancedb/lancedb-linux-x64-musl", + "version": "0.13.0", + "os": ["linux"], + "cpu": ["x64"], + "main": "lancedb.linux-x64-musl.node", + "files": ["lancedb.linux-x64-musl.node"], + "license": "Apache 2.0", + "engines": { + "node": ">= 18" + }, + "libc": ["musl"] +} \ No newline at end of file diff --git a/nodejs/package.json b/nodejs/package.json index 234c8960..90a04b72 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -24,10 +24,12 @@ "triples": { "defaults": false, "additional": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", + "aarch64-apple-darwin", "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", "x86_64-pc-windows-msvc" ] }