mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-07 04:12:59 +00:00
0a1ea1858d
```shell git rm -r rust/ffi git rm -r node git rm ci/build_windows_artifacts.ps1 git rm ci/build_windows_artifacts_nodejs.ps1 git rm ci/build_linux_artifacts.sh git rm ci/build_macos_artifacts.sh git rm -r ci/manylinux_node git rm .github/workflows/node.yml ```
368 lines
13 KiB
YAML
368 lines
13 KiB
YAML
name: NPM Publish
|
|
|
|
env:
|
|
MACOSX_DEPLOYMENT_TARGET: '10.13'
|
|
CARGO_INCREMENTAL: '0'
|
|
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
pull_request:
|
|
# This should trigger a dry run (we skip the final publish step)
|
|
paths:
|
|
- .github/workflows/npm-publish.yml
|
|
- Cargo.toml # Change in dependency frequently breaks builds
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
gh-release:
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Extract version
|
|
id: extract_version
|
|
env:
|
|
GITHUB_REF: ${{ github.ref }}
|
|
run: |
|
|
set -e
|
|
echo "Extracting tag and version from $GITHUB_REF"
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
|
|
VERSION=${BASH_REMATCH[1]}
|
|
TAG=v$VERSION
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Failed to extract version from $GITHUB_REF"
|
|
exit 1
|
|
fi
|
|
echo "Extracted version $VERSION from $GITHUB_REF"
|
|
if [[ $VERSION =~ beta ]]; then
|
|
echo "This is a beta release"
|
|
|
|
# Get last release (that is not this one)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
else
|
|
echo "This is a stable release"
|
|
# Get last stable tag (ignore betas)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| grep -v beta \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
fi
|
|
echo "Found from tag $FROM_TAG"
|
|
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
|
|
- name: Create Release Notes
|
|
id: release_notes
|
|
uses: mikepenz/release-changelog-builder-action@v4
|
|
with:
|
|
configuration: .github/release_notes.json
|
|
toTag: ${{ steps.extract_version.outputs.tag }}
|
|
fromTag: ${{ steps.extract_version.outputs.from_tag }}
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: Create GH release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
prerelease: ${{ contains('beta', github.ref) }}
|
|
tag_name: ${{ steps.extract_version.outputs.tag }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
generate_release_notes: false
|
|
name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
|
|
body: ${{ steps.release_notes.outputs.changelog }}
|
|
|
|
build-lancedb:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
- target: x86_64-apple-darwin
|
|
host: macos-latest
|
|
features: ","
|
|
pre_build: |-
|
|
brew install protobuf
|
|
rustup target add x86_64-apple-darwin
|
|
- target: aarch64-apple-darwin
|
|
host: macos-latest
|
|
features: fp16kernels
|
|
pre_build: brew install protobuf
|
|
- target: x86_64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc ninja nasm
|
|
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
|
|
# There is an issue where choco doesn't add nasm to the path
|
|
export PATH="$PATH:/c/Program Files/NASM"
|
|
nasm -v
|
|
- target: aarch64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc
|
|
rustup target add aarch64-pc-windows-msvc
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
features: fp16kernels
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config
|
|
- target: x86_64-unknown-linux-musl
|
|
# This one seems to need some extra memory
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: fp16kernels
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev curl &&
|
|
ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
|
|
ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
|
|
CC=gcc &&
|
|
CXX=g++
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
|
|
features: "fp16kernels"
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config &&
|
|
# https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
|
|
rustup target add aarch64-unknown-linux-gnu
|
|
- target: aarch64-unknown-linux-musl
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: ","
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev &&
|
|
rustup target add aarch64-unknown-linux-musl &&
|
|
export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
|
|
export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
|
|
name: build - ${{ matrix.settings.target }}
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install
|
|
uses: dtolnay/rust-toolchain@stable
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
toolchain: stable
|
|
targets: ${{ matrix.settings.target }}
|
|
- name: Cache cargo
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cargo/registry/index/
|
|
~/.cargo/registry/cache/
|
|
~/.cargo/git/db/
|
|
.cargo-cache
|
|
target/
|
|
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
|
- name: Setup toolchain
|
|
run: ${{ matrix.settings.setup }}
|
|
if: ${{ matrix.settings.setup }}
|
|
shell: bash
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Build in docker
|
|
uses: addnab/docker-run-action@v3
|
|
if: ${{ matrix.settings.docker }}
|
|
with:
|
|
image: ${{ matrix.settings.docker }}
|
|
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
|
|
-v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
|
|
-v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
|
|
-v ${{ github.workspace }}:/build -w /build/nodejs"
|
|
run: |
|
|
set -e
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
dist/
|
|
- name: Build
|
|
run: |
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
$EXTRA_ARGS \
|
|
dist/
|
|
if: ${{ !matrix.settings.docker }}
|
|
shell: bash
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/*.node
|
|
if-no-files-found: error
|
|
# The generic files are the same in all distros so we just pick
|
|
# one to do the upload.
|
|
- name: Make generic artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
run: npm run tsc
|
|
- name: Upload Generic Artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: |
|
|
nodejs/dist/*
|
|
!nodejs/dist/*.node
|
|
test-lancedb:
|
|
name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
|
|
needs:
|
|
- build-lancedb
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
# TODO: Get tests passing on Windows (failing from test tmpdir issue)
|
|
# - host: windows-latest
|
|
# target: x86_64-pc-windows-msvc
|
|
- host: macos-latest
|
|
target: aarch64-apple-darwin
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: buildjet-16vcpu-ubuntu-2204-arm
|
|
node:
|
|
- '20'
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ matrix.node }}
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Download artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
# run-id: 13982782871
|
|
- name: List packages
|
|
run: ls -R dist
|
|
- name: Move built files
|
|
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
|
- name: Test bindings
|
|
run: npm test
|
|
publish:
|
|
name: Publish
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
needs:
|
|
- test-lancedb
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
registry-url: "https://registry.npmjs.org"
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
name: Download arch-specific binaries
|
|
with:
|
|
pattern: lancedb-*
|
|
path: nodejs/nodejs-artifacts
|
|
merge-multiple: true
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- name: Display structure of downloaded files
|
|
run: find dist && find nodejs-artifacts
|
|
- name: Move artifacts
|
|
run: npx napi artifacts -d nodejs-artifacts
|
|
- name: List packages
|
|
run: find npm
|
|
- name: Publish
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
|
|
DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
run: |
|
|
ARGS="--access public"
|
|
if [[ $DRY_RUN == "true" ]]; then
|
|
ARGS="$ARGS --dry-run"
|
|
fi
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
|
|
ARGS="$ARGS --tag preview"
|
|
fi
|
|
npm publish $ARGS
|