mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-22 21:09:58 +00:00
3cd73f9f5a
We have very low download stats for mac x86, and also latest github runners for mac are all arm, so it makes sense at this point to deprecate x86 support in general.
376 lines
13 KiB
YAML
376 lines
13 KiB
YAML
name: NPM Publish
|
|
|
|
env:
|
|
MACOSX_DEPLOYMENT_TARGET: '10.13'
|
|
CARGO_INCREMENTAL: '0'
|
|
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
pull_request:
|
|
# This should trigger a dry run (we skip the final publish step)
|
|
paths:
|
|
- .github/workflows/npm-publish.yml
|
|
- Cargo.toml # Change in dependency frequently breaks builds
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
gh-release:
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Extract version
|
|
id: extract_version
|
|
env:
|
|
GITHUB_REF: ${{ github.ref }}
|
|
run: |
|
|
set -e
|
|
echo "Extracting tag and version from $GITHUB_REF"
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
|
|
VERSION=${BASH_REMATCH[1]}
|
|
TAG=v$VERSION
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Failed to extract version from $GITHUB_REF"
|
|
exit 1
|
|
fi
|
|
echo "Extracted version $VERSION from $GITHUB_REF"
|
|
if [[ $VERSION =~ beta ]]; then
|
|
echo "This is a beta release"
|
|
|
|
# Get last release (that is not this one)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
else
|
|
echo "This is a stable release"
|
|
# Get last stable tag (ignore betas)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| grep -v beta \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
fi
|
|
echo "Found from tag $FROM_TAG"
|
|
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
|
|
- name: Create Release Notes
|
|
id: release_notes
|
|
uses: mikepenz/release-changelog-builder-action@v4
|
|
with:
|
|
configuration: .github/release_notes.json
|
|
toTag: ${{ steps.extract_version.outputs.tag }}
|
|
fromTag: ${{ steps.extract_version.outputs.from_tag }}
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: Create GH release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
prerelease: ${{ contains('beta', github.ref) }}
|
|
tag_name: ${{ steps.extract_version.outputs.tag }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
generate_release_notes: false
|
|
name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
|
|
body: ${{ steps.release_notes.outputs.changelog }}
|
|
|
|
build-lancedb:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
- target: aarch64-apple-darwin
|
|
host: macos-latest
|
|
features: fp16kernels
|
|
pre_build: brew install protobuf
|
|
- target: x86_64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc ninja nasm
|
|
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
|
|
# There is an issue where choco doesn't add nasm to the path
|
|
export PATH="$PATH:/c/Program Files/NASM"
|
|
nasm -v
|
|
- target: aarch64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc
|
|
rustup target add aarch64-pc-windows-msvc
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
features: fp16kernels
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config
|
|
- target: x86_64-unknown-linux-musl
|
|
# This one seems to need some extra memory
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: fp16kernels
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev curl &&
|
|
ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
|
|
ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
|
|
CC=gcc &&
|
|
CXX=g++
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
|
|
features: "fp16kernels"
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config &&
|
|
# https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
|
|
rustup target add aarch64-unknown-linux-gnu
|
|
- target: aarch64-unknown-linux-musl
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: ","
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev &&
|
|
rustup target add aarch64-unknown-linux-musl &&
|
|
export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
|
|
export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
|
|
name: build - ${{ matrix.settings.target }}
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install
|
|
uses: dtolnay/rust-toolchain@stable
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
toolchain: stable
|
|
targets: ${{ matrix.settings.target }}
|
|
- name: Cache cargo
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cargo/registry/index/
|
|
~/.cargo/registry/cache/
|
|
~/.cargo/git/db/
|
|
.cargo-cache
|
|
target/
|
|
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
|
- name: Setup toolchain
|
|
run: ${{ matrix.settings.setup }}
|
|
if: ${{ matrix.settings.setup }}
|
|
shell: bash
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Build in docker
|
|
uses: addnab/docker-run-action@v3
|
|
if: ${{ matrix.settings.docker }}
|
|
with:
|
|
image: ${{ matrix.settings.docker }}
|
|
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
|
|
-v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
|
|
-v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
|
|
-v ${{ github.workspace }}:/build -w /build/nodejs"
|
|
run: |
|
|
set -e
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
dist/
|
|
- name: Build
|
|
run: |
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
$EXTRA_ARGS \
|
|
dist/
|
|
if: ${{ !matrix.settings.docker }}
|
|
shell: bash
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/*.node
|
|
if-no-files-found: error
|
|
# The generic files are the same in all distros so we just pick
|
|
# one to do the upload.
|
|
- name: Make generic artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
run: npm run tsc
|
|
- name: Upload Generic Artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: |
|
|
nodejs/dist/*
|
|
!nodejs/dist/*.node
|
|
test-lancedb:
|
|
name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
|
|
needs:
|
|
- build-lancedb
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
# TODO: Get tests passing on Windows (failing from test tmpdir issue)
|
|
# - host: windows-latest
|
|
# target: x86_64-pc-windows-msvc
|
|
- host: macos-latest
|
|
target: aarch64-apple-darwin
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: buildjet-16vcpu-ubuntu-2204-arm
|
|
node:
|
|
- '20'
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ matrix.node }}
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Download artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
# run-id: 13982782871
|
|
- name: List packages
|
|
run: ls -R dist
|
|
- name: Move built files
|
|
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
|
- name: Test bindings
|
|
run: npm test
|
|
publish:
|
|
name: Publish
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
needs:
|
|
- test-lancedb
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
registry-url: "https://registry.npmjs.org"
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
name: Download arch-specific binaries
|
|
with:
|
|
pattern: lancedb-*
|
|
path: nodejs/nodejs-artifacts
|
|
merge-multiple: true
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- name: Display structure of downloaded files
|
|
run: find dist && find nodejs-artifacts
|
|
- name: Move artifacts
|
|
run: npx napi artifacts -d nodejs-artifacts
|
|
- name: List packages
|
|
run: find npm
|
|
- name: Publish
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
|
|
DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
run: |
|
|
ARGS="--access public"
|
|
if [[ $DRY_RUN == "true" ]]; then
|
|
ARGS="$ARGS --dry-run"
|
|
fi
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
|
|
ARGS="$ARGS --tag preview"
|
|
fi
|
|
npm publish $ARGS
|
|
report-failure:
|
|
name: Report Workflow Failure
|
|
runs-on: ubuntu-latest
|
|
needs: [build-lancedb, test-lancedb, publish]
|
|
if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: ./.github/actions/create-failure-issue
|
|
with:
|
|
job-results: ${{ toJSON(needs) }}
|
|
workflow-name: ${{ github.workflow }}
|