mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-04 10:52:56 +00:00
4a2cdbf299
This should prevent the failures we are seeing in Node release. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Chore** - Enhanced the package deprecation process with improved security measures, ensuring smoother and more reliable updates during package deprecation. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
567 lines
19 KiB
YAML
567 lines
19 KiB
YAML
name: NPM Publish
|
|
|
|
env:
|
|
MACOSX_DEPLOYMENT_TARGET: '10.13'
|
|
CARGO_INCREMENTAL: '0'
|
|
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
pull_request:
|
|
# This should trigger a dry run (we skip the final publish step)
|
|
paths:
|
|
- .github/workflows/npm-publish.yml
|
|
- Cargo.toml # Change in dependency frequently breaks builds
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
gh-release:
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- name: Extract version
|
|
id: extract_version
|
|
env:
|
|
GITHUB_REF: ${{ github.ref }}
|
|
run: |
|
|
set -e
|
|
echo "Extracting tag and version from $GITHUB_REF"
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
|
|
VERSION=${BASH_REMATCH[1]}
|
|
TAG=v$VERSION
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "Failed to extract version from $GITHUB_REF"
|
|
exit 1
|
|
fi
|
|
echo "Extracted version $VERSION from $GITHUB_REF"
|
|
if [[ $VERSION =~ beta ]]; then
|
|
echo "This is a beta release"
|
|
|
|
# Get last release (that is not this one)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
else
|
|
echo "This is a stable release"
|
|
# Get last stable tag (ignore betas)
|
|
FROM_TAG=$(git tag --sort='version:refname' \
|
|
| grep ^v \
|
|
| grep -vF "$TAG" \
|
|
| grep -v beta \
|
|
| python ci/semver_sort.py v \
|
|
| tail -n 1)
|
|
fi
|
|
echo "Found from tag $FROM_TAG"
|
|
echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
|
|
- name: Create Release Notes
|
|
id: release_notes
|
|
uses: mikepenz/release-changelog-builder-action@v4
|
|
with:
|
|
configuration: .github/release_notes.json
|
|
toTag: ${{ steps.extract_version.outputs.tag }}
|
|
fromTag: ${{ steps.extract_version.outputs.from_tag }}
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- name: Create GH release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
prerelease: ${{ contains('beta', github.ref) }}
|
|
tag_name: ${{ steps.extract_version.outputs.tag }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
generate_release_notes: false
|
|
name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
|
|
body: ${{ steps.release_notes.outputs.changelog }}
|
|
|
|
build-lancedb:
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
- target: x86_64-apple-darwin
|
|
host: macos-latest
|
|
features: ","
|
|
pre_build: |-
|
|
brew install protobuf
|
|
rustup target add x86_64-apple-darwin
|
|
- target: aarch64-apple-darwin
|
|
host: macos-latest
|
|
features: fp16kernels
|
|
pre_build: brew install protobuf
|
|
- target: x86_64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc ninja nasm
|
|
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
|
|
# There is an issue where choco doesn't add nasm to the path
|
|
export PATH="$PATH:/c/Program Files/NASM"
|
|
nasm -v
|
|
- target: aarch64-pc-windows-msvc
|
|
host: windows-latest
|
|
features: ","
|
|
pre_build: |-
|
|
choco install --no-progress protoc
|
|
rustup target add aarch64-pc-windows-msvc
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
features: fp16kernels
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config
|
|
- target: x86_64-unknown-linux-musl
|
|
# This one seems to need some extra memory
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: fp16kernels
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev curl &&
|
|
ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
|
|
ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
|
|
CC=gcc &&
|
|
CXX=g++
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
|
|
features: "fp16kernels"
|
|
pre_build: |-
|
|
set -e &&
|
|
apt-get update &&
|
|
apt-get install -y protobuf-compiler pkg-config &&
|
|
# https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
|
|
ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
|
|
rustup target add aarch64-unknown-linux-gnu
|
|
- target: aarch64-unknown-linux-musl
|
|
host: ubuntu-2404-8x-x64
|
|
# https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
|
|
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
|
features: ","
|
|
pre_build: |-
|
|
set -e &&
|
|
apk add protobuf-dev &&
|
|
rustup target add aarch64-unknown-linux-musl &&
|
|
export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
|
|
export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
|
|
name: build - ${{ matrix.settings.target }}
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install
|
|
uses: dtolnay/rust-toolchain@stable
|
|
if: ${{ !matrix.settings.docker }}
|
|
with:
|
|
toolchain: stable
|
|
targets: ${{ matrix.settings.target }}
|
|
- name: Cache cargo
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: |
|
|
~/.cargo/registry/index/
|
|
~/.cargo/registry/cache/
|
|
~/.cargo/git/db/
|
|
.cargo-cache
|
|
target/
|
|
key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
|
|
- name: Setup toolchain
|
|
run: ${{ matrix.settings.setup }}
|
|
if: ${{ matrix.settings.setup }}
|
|
shell: bash
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Build in docker
|
|
uses: addnab/docker-run-action@v3
|
|
if: ${{ matrix.settings.docker }}
|
|
with:
|
|
image: ${{ matrix.settings.docker }}
|
|
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
|
|
-v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
|
|
-v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
|
|
-v ${{ github.workspace }}:/build -w /build/nodejs"
|
|
run: |
|
|
set -e
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
dist/
|
|
- name: Build
|
|
run: |
|
|
${{ matrix.settings.pre_build }}
|
|
npx napi build --platform --release --no-const-enum \
|
|
--features ${{ matrix.settings.features }} \
|
|
--target ${{ matrix.settings.target }} \
|
|
--dts ../lancedb/native.d.ts \
|
|
--js ../lancedb/native.js \
|
|
--strip \
|
|
$EXTRA_ARGS \
|
|
dist/
|
|
if: ${{ !matrix.settings.docker }}
|
|
shell: bash
|
|
- name: Upload artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/*.node
|
|
if-no-files-found: error
|
|
# The generic files are the same in all distros so we just pick
|
|
# one to do the upload.
|
|
- name: Make generic artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
run: npm run tsc
|
|
- name: Upload Generic Artifacts
|
|
if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: |
|
|
nodejs/dist/*
|
|
!nodejs/dist/*.node
|
|
test-lancedb:
|
|
name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
|
|
needs:
|
|
- build-lancedb
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
settings:
|
|
# TODO: Get tests passing on Windows (failing from test tmpdir issue)
|
|
# - host: windows-latest
|
|
# target: x86_64-pc-windows-msvc
|
|
- host: macos-latest
|
|
target: aarch64-apple-darwin
|
|
- target: x86_64-unknown-linux-gnu
|
|
host: ubuntu-latest
|
|
- target: aarch64-unknown-linux-gnu
|
|
host: buildjet-16vcpu-ubuntu-2204-arm
|
|
node:
|
|
- '20'
|
|
runs-on: ${{ matrix.settings.host }}
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: ${{ matrix.node }}
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- name: Download artifacts
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: lancedb-${{ matrix.settings.target }}
|
|
path: nodejs/dist/
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
# run-id: 13982782871
|
|
- name: List packages
|
|
run: ls -R dist
|
|
- name: Move built files
|
|
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
|
|
- name: Test bindings
|
|
run: npm test
|
|
publish:
|
|
name: Publish
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: nodejs
|
|
needs:
|
|
- test-lancedb
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
cache: npm
|
|
cache-dependency-path: nodejs/package-lock.json
|
|
registry-url: "https://registry.npmjs.org"
|
|
- name: Install dependencies
|
|
run: npm ci
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
name: nodejs-dist
|
|
path: nodejs/dist
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- uses: actions/download-artifact@v4
|
|
name: Download arch-specific binaries
|
|
with:
|
|
pattern: lancedb-*
|
|
path: nodejs/nodejs-artifacts
|
|
merge-multiple: true
|
|
# For testing purposes:
|
|
# run-id: 13982782871
|
|
# github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
|
|
- name: Display structure of downloaded files
|
|
run: find dist && find nodejs-artifacts
|
|
- name: Move artifacts
|
|
run: npx napi artifacts -d nodejs-artifacts
|
|
- name: List packages
|
|
run: find npm
|
|
- name: Publish
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
|
|
DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
|
|
run: |
|
|
ARGS="--access public"
|
|
if [[ $DRY_RUN == "true" ]]; then
|
|
ARGS="$ARGS --dry-run"
|
|
fi
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
|
|
ARGS="$ARGS --tag preview"
|
|
fi
|
|
npm publish $ARGS
|
|
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# vectordb release (legacy)
|
|
# ----------------------------------------------------------------------------
|
|
# TODO: delete this when we drop vectordb
|
|
node:
|
|
name: vectordb Typescript
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
working-directory: node
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- uses: actions/setup-node@v3
|
|
with:
|
|
node-version: 20
|
|
cache: "npm"
|
|
cache-dependency-path: node/package-lock.json
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt update
|
|
sudo apt install -y protobuf-compiler libssl-dev
|
|
- name: Build
|
|
run: |
|
|
npm ci
|
|
npm run tsc
|
|
npm pack
|
|
- name: Upload Linux Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: node-package
|
|
path: |
|
|
node/vectordb-*.tgz
|
|
|
|
node-macos:
|
|
name: vectordb ${{ matrix.config.arch }}
|
|
strategy:
|
|
matrix:
|
|
config:
|
|
- arch: x86_64-apple-darwin
|
|
runner: macos-13
|
|
- arch: aarch64-apple-darwin
|
|
# xlarge is implicitly arm64.
|
|
runner: macos-14
|
|
runs-on: ${{ matrix.config.runner }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Install system dependencies
|
|
run: brew install protobuf
|
|
- name: Install npm dependencies
|
|
run: |
|
|
cd node
|
|
npm ci
|
|
- name: Build MacOS native node modules
|
|
run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
|
|
- name: Upload Darwin Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: node-native-darwin-${{ matrix.config.arch }}
|
|
path: |
|
|
node/dist/lancedb-vectordb-darwin*.tgz
|
|
|
|
node-linux-gnu:
|
|
name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
|
|
runs-on: ${{ matrix.config.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
config:
|
|
- arch: x86_64
|
|
runner: ubuntu-latest
|
|
- arch: aarch64
|
|
# For successful fat LTO builds, we need a large runner to avoid OOM errors.
|
|
runner: warp-ubuntu-latest-arm64-4x
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
# To avoid OOM errors on ARM, we create a swap file.
|
|
- name: Configure aarch64 build
|
|
if: ${{ matrix.config.arch == 'aarch64' }}
|
|
run: |
|
|
free -h
|
|
sudo fallocate -l 16G /swapfile
|
|
sudo chmod 600 /swapfile
|
|
sudo mkswap /swapfile
|
|
sudo swapon /swapfile
|
|
echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
|
|
# print info
|
|
swapon --show
|
|
free -h
|
|
- name: Build Linux Artifacts
|
|
run: |
|
|
bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
|
|
- name: Upload Linux Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: node-native-linux-${{ matrix.config.arch }}-gnu
|
|
path: |
|
|
node/dist/lancedb-vectordb-linux*.tgz
|
|
|
|
node-windows:
|
|
name: vectordb ${{ matrix.target }}
|
|
runs-on: windows-2022
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
target: [x86_64-pc-windows-msvc]
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Install Protoc v21.12
|
|
working-directory: C:\
|
|
run: |
|
|
New-Item -Path 'C:\protoc' -ItemType Directory
|
|
Set-Location C:\protoc
|
|
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
|
|
7z x protoc.zip
|
|
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
|
|
shell: powershell
|
|
- name: Install npm dependencies
|
|
run: |
|
|
cd node
|
|
npm ci
|
|
- name: Build Windows native node modules
|
|
run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
|
|
- name: Upload Windows Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: node-native-windows
|
|
path: |
|
|
node/dist/lancedb-vectordb-win32*.tgz
|
|
|
|
release:
|
|
name: vectordb NPM Publish
|
|
needs: [node, node-macos, node-linux-gnu, node-windows]
|
|
runs-on: ubuntu-latest
|
|
# Only runs on tags that matches the make-release action
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
steps:
|
|
- uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: node-*
|
|
- name: Display structure of downloaded files
|
|
run: ls -R
|
|
- uses: actions/setup-node@v3
|
|
with:
|
|
node-version: 20
|
|
registry-url: "https://registry.npmjs.org"
|
|
- name: Publish to NPM
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
|
|
run: |
|
|
# Tag beta as "preview" instead of default "latest". See lancedb
|
|
# npm publish step for more info.
|
|
if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
|
|
PUBLISH_ARGS="--tag preview"
|
|
fi
|
|
|
|
mv */*.tgz .
|
|
for filename in *.tgz; do
|
|
npm publish $PUBLISH_ARGS $filename
|
|
done
|
|
- name: Deprecate
|
|
env:
|
|
NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
|
|
# We need to deprecate the old package to avoid confusion.
|
|
# Each time we publish a new version, it gets undeprecated.
|
|
run: npm deprecate vectordb "Use @lancedb/lancedb instead."
|
|
- name: Notify Slack Action
|
|
uses: ravsamhq/notify-slack-action@2.3.0
|
|
if: ${{ always() }}
|
|
with:
|
|
status: ${{ job.status }}
|
|
notify_when: "failure"
|
|
notification_title: "{workflow} is failing"
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
|
|
|
|
update-package-lock:
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
needs: [release]
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: main
|
|
token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
|
|
fetch-depth: 0
|
|
lfs: true
|
|
- uses: ./.github/workflows/update_package_lock
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|