mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
Will Jones
0d9c87a079
The `build - aarch64-pc-windows-msvc` node build job (and, marginally, the x86_64 one) had started hitting `rustc-LLVM ERROR: out of memory` while linking the `lancedb-nodejs` cdylib — most recently surfaced by #3526, which adds the goosefs backend (and its tonic/prost gRPC subtree) to the default node binary. The peak-memory step is the fat-LTO codegen (`lto=fat`, `codegen-units=1` from `.cargo/config.toml`), which merges the whole crate graph into a single LLVM module and runs single-threaded. It therefore neither parallelizes across cores nor fits in the 16 GB of the standard `windows-latest` runner as the dependency graph grows. This PR: - Moves both `*-pc-windows-msvc` node build jobs to `windows-2025-8x-x64` (more memory + cores). - Overrides the release profile to ThinLTO for just these jobs, via `CARGO_PROFILE_RELEASE_LTO=thin` / `CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16` in `pre_build`. ThinLTO parallelizes the cross-module optimization across the runner's cores and keeps peak memory well under the limit. Scoped so Python wheels and Rust release builds keep fat LTO. The larger runner alone would clear the OOM but waste the added cores on the single-threaded fat-LTO tail; ThinLTO is what makes the extra cores actually reduce wall-clock and gives durable memory headroom for future dependency growth. Tradeoff: ThinLTO can leave a small runtime-perf gap vs fat LTO for the node native binary, but it recovers most of it and is a common release configuration. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
415 lines
15 KiB
YAML
415 lines
15 KiB
YAML
name: NPM Publish
|
|
|
|
env:
|
|
MACOSX_DEPLOYMENT_TARGET: '10.13'
|
|
CARGO_INCREMENTAL: '0'
|
|
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
pull_request:
|
|
# This should trigger a dry run (we skip the final publish step)
|
|
paths:
|
|
- .github/workflows/npm-publish.yml
|
|
- Cargo.toml # Change in dependency frequently breaks builds
|
|
- Cargo.lock
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
gh-release:
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Extract version
|
|
id: extract_version
|
|
env:
|
|
GITHUB_REF: ${{ github.ref }}
|
|
run: |
|
|
set -e
|
|
echo "Extracting tag and version from $GITHUB_REF"
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
|
|
VERSION=${BASH_REMATCH[1]}
|
|
TAG=v$VERSION
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Failed to extract version from $GITHUB_REF"
|
|
exit 1
|
|
fi
|
|
echo "Extracted version $VERSION from $GITHUB_REF"
|
|
if [[ $VERSION =~ beta ]]; then
|
|
echo "This is a beta release"
|
|
|
|
# Get last release (that is not this one)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
else
|
|
echo "This is a stable release"
|
|
# Get last stable tag (ignore betas)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| grep -v beta \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
fi
|
|
echo "Found from tag $FROM_TAG"
|
|
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
|
|
- name: Create Release Notes
|
|
id: release_notes
|
|
uses: mikepenz/release-changelog-builder-action@v4
|
|
with:
|
|
configuration: .github/release_notes.json
|
|
toTag: ${{ steps.extract_version.outputs.tag }}
|
|
fromTag: ${{ steps.extract_version.outputs.from_tag }}
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: Create GH release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
prerelease: ${{ contains('beta', github.ref) }}
|
|
tag_name: ${{ steps.extract_version.outputs.tag }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
generate_release_notes: false
|
|
name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
|
|
body: ${{ steps.release_notes.outputs.changelog }}
|
|
|
|
build-lancedb:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
- target: aarch64-apple-darwin
|
|
host: macos-latest
|
|
features: fp16kernels
|
|
pre_build: brew install protobuf
|
|
- target: x86_64-pc-windows-msvc
|
|
host: windows-2025-8x-x64
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc ninja nasm
|
|
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
|
|
# There is an issue where choco doesn't add nasm to the path
|
|
export PATH="$PATH:/c/Program Files/NASM"
|
|
nasm -v
|
|
# Fat LTO of the cdylib is single-threaded and the peak-memory
|
|
# step of the build, and had started hitting rustc-LLVM OOM on the
|
|
# Windows runners. ThinLTO parallelizes it across the runner's
|
|
# cores and keeps peak memory well under the limit.
|
|
export CARGO_PROFILE_RELEASE_LTO=thin
|
|
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
|
- target: aarch64-pc-windows-msvc
|
|
host: windows-2025-8x-x64
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc
|
|
rustup target add aarch64-pc-windows-msvc
|
|
# See ThinLTO note on the x86_64-pc-windows-msvc target above.
|
|
export CARGO_PROFILE_RELEASE_LTO=thin
|
|
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
features: fp16kernels
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config &&
|
|
# The base image (manylinux2014-cross) sets TARGET_CC to the old
|
|
# GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
|
|
# so it picks up GCC even though the napi-rs image sets CC=clang.
|
|
# Override to use the image's clang-18 which supports -fuse-ld=lld.
|
|
export TARGET_CC=clang TARGET_CXX=clang++
|
|
- target: x86_64-unknown-linux-musl
|
|
# This one seems to need some extra memory
|
|
host: ubuntu-2404-8x-x64
|
|
features: fp16kernels
|
|
pre_build: |-
|
|
set -e &&
|
|
sudo apt-get update &&
|
|
sudo apt-get install -y protobuf-compiler pkg-config &&
|
|
rustup target add x86_64-unknown-linux-musl &&
|
|
export EXTRA_ARGS="-x"
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
|
|
features: "fp16kernels"
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config &&
|
|
export TARGET_CC=clang TARGET_CXX=clang++ &&
|
|
# The manylinux2014 sysroot has glibc 2.17 headers which lack
|
|
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
|
|
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
|
|
rustup target add aarch64-unknown-linux-gnu
|
|
- target: aarch64-unknown-linux-musl
|
|
host: ubuntu-2404-8x-x64
|
|
features: ","
|
|
pre_build: |-
|
|
set -e &&
|
|
sudo apt-get update &&
|
|
sudo apt-get install -y protobuf-compiler &&
|
|
rustup target add aarch64-unknown-linux-musl &&
|
|
export EXTRA_ARGS="-x"
|
|
name: build - ${{ matrix.settings.target }}
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@v6
|
|
with:
|
|
version: 11.1.1
|
|
- name: Setup node
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
|
# in October.
|
|
node-version: 24
|
|
cache: pnpm
|
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
|
- name: Install
|
|
uses: dtolnay/rust-toolchain@stable
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
toolchain: stable
|
|
targets: ${{ matrix.settings.target }}
|
|
- name: Cache cargo
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: |
|
|
~/.cargo/registry/index/
|
|
~/.cargo/registry/cache/
|
|
~/.cargo/git/db/
|
|
.cargo-cache
|
|
target/
|
|
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
|
- name: Install dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
- name: Install Zig
|
|
uses: mlugg/setup-zig@v2
|
|
if: ${{ contains(matrix.settings.target, 'musl') }}
|
|
with:
|
|
version: 0.14.1
|
|
- name: Install cargo-zigbuild
|
|
uses: taiki-e/install-action@v2
|
|
if: ${{ contains(matrix.settings.target, 'musl') }}
|
|
with:
|
|
tool: cargo-zigbuild
|
|
- name: Build in docker
|
|
uses: addnab/docker-run-action@v3
|
|
if: ${{ matrix.settings.docker }}
|
|
with:
|
|
image: ${{ matrix.settings.docker }}
|
|
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
|
|
-v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
|
|
-v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
|
|
-v ${{ github.workspace }}:/build -w /build/nodejs"
|
|
run: |
|
|
set -e
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
--output-dir dist/
|
|
- name: Build
|
|
run: |
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
$EXTRA_ARGS \
|
|
--output-dir dist/
|
|
if: ${{ !matrix.settings.docker }}
|
|
shell: bash
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/*.node
|
|
if-no-files-found: error
|
|
# The generic files are the same in all distros so we just pick
|
|
# one to do the upload.
|
|
- name: Make generic artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
run: pnpm tsc
|
|
- name: Upload Generic Artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: nodejs-dist
|
|
path: |
|
|
nodejs/dist/*
|
|
!nodejs/dist/*.node
|
|
test-lancedb:
|
|
name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
|
|
needs:
|
|
- build-lancedb
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
# TODO: Get tests passing on Windows (failing from test tmpdir issue)
|
|
# - host: windows-latest
|
|
# target: x86_64-pc-windows-msvc
|
|
- host: macos-latest
|
|
target: aarch64-apple-darwin
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: ubuntu-2404-8x-arm64
|
|
node:
|
|
- '20'
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@v6
|
|
with:
|
|
version: 11.1.1
|
|
- name: Setup Node.js 24 for install
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
|
|
# in October.
|
|
node-version: 24
|
|
cache: pnpm
|
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
|
- name: Install dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
- name: Setup Node.js ${{ matrix.node }} for test
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: ${{ matrix.node }}
|
|
- name: Download artifacts
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
# run-id: 13982782871
|
|
- name: List packages
|
|
run: ls -R dist
|
|
- name: Move built files
|
|
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
|
- name: Test bindings
|
|
# Invoke jest directly because pnpm 11 itself requires Node 22+
|
|
# while the matrix tests on older Node versions.
|
|
run: npx jest --verbose
|
|
publish:
|
|
name: Publish
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
needs:
|
|
- test-lancedb
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- name: Setup pnpm
|
|
uses: pnpm/action-setup@v6
|
|
with:
|
|
version: 11.1.1
|
|
- name: Setup node
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: 24
|
|
cache: pnpm
|
|
cache-dependency-path: nodejs/pnpm-lock.yaml
|
|
registry-url: "https://registry.npmjs.org"
|
|
- name: Install dependencies
|
|
run: pnpm install --frozen-lockfile
|
|
- uses: actions/download-artifact@v8
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v8
|
|
name: Download arch-specific binaries
|
|
with:
|
|
pattern: lancedb-*
|
|
path: nodejs/nodejs-artifacts
|
|
merge-multiple: true
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- name: Display structure of downloaded files
|
|
run: find dist && find nodejs-artifacts
|
|
- name: Move artifacts
|
|
run: pnpm exec napi artifacts -d nodejs-artifacts
|
|
- name: List packages
|
|
run: find npm
|
|
- name: Publish
|
|
env:
|
|
DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
run: |
|
|
npm config set provenance true
|
|
ARGS="--access public"
|
|
if [[ $DRY_RUN == "true" ]]; then
|
|
ARGS="$ARGS --dry-run"
|
|
fi
|
|
VERSION=$(node -p "require('./package.json').version")
|
|
if [[ $VERSION == *-* ]]; then
|
|
ARGS="$ARGS --tag preview"
|
|
fi
|
|
npm publish $ARGS
|
|
report-failure:
|
|
name: Report Workflow Failure
|
|
runs-on: ubuntu-latest
|
|
needs: [build-lancedb, test-lancedb, publish]
|
|
if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
- uses: ./.github/actions/create-failure-issue
|
|
with:
|
|
job-results: ${{ toJSON(needs) }}
|
|
workflow-name: ${{ github.workflow }}
|