lancedb/.github/workflows/npm-publish.yml

name: NPM Publish

env:
  MACOSX_DEPLOYMENT_TARGET: '10.13'
  CARGO_INCREMENTAL: '0'

permissions:
  contents: write
  id-token: write

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/npm-publish.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  gh-release:
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Release Notes
        id: release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.release_notes.outputs.changelog }}

  build-lancedb:
    strategy:
      fail-fast: false
      matrix:
        settings:
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
            pre_build: brew install protobuf
          - target: x86_64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
          - target: aarch64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              # The base image (manylinux2014-cross) sets TARGET_CC to the old
              # GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
              # so it picks up GCC even though the napi-rs image sets CC=clang.
              # Override to use the image's clang-18 which supports -fuse-ld=lld.
              export TARGET_CC=clang TARGET_CXX=clang++
          - target: x86_64-unknown-linux-musl
            # This one seems to need some extra memory
            host: ubuntu-2404-8x-x64
            features: fp16kernels
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler pkg-config &&
              rustup target add x86_64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              export TARGET_CC=clang TARGET_CXX=clang++ &&
              # The manylinux2014 sysroot has glibc 2.17 headers which lack
              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
              rustup target add aarch64-unknown-linux-gnu
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            features: ","
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler &&
              rustup target add aarch64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 11.1.1
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
          # in October.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
      - name: Install
        uses: dtolnay/rust-toolchain@stable
        if: ${{ !matrix.settings.docker }}
        with:
          toolchain: stable
          targets: ${{ matrix.settings.target }}
      - name: Cache cargo
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - name: Install Zig
        uses: mlugg/setup-zig@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          version: 0.14.1
      - name: Install cargo-zigbuild
        uses: taiki-e/install-action@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          tool: cargo-zigbuild
      - name: Build in docker
        uses: addnab/docker-run-action@v3
        if: ${{ matrix.settings.docker }}
        with:
          image: ${{ matrix.settings.docker }}
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
            npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              --output-dir dist/
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
          npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              $EXTRA_ARGS \
              --output-dir dist/
        if: ${{ !matrix.settings.docker }}
        shell: bash
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/*.node
          if-no-files-found: error
      # The generic files are the same in all distros so we just pick
      # one to do the upload.
      - name: Make generic artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        run: pnpm tsc
      - name: Upload Generic Artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-dist
          path: |
            nodejs/dist/*
            !nodejs/dist/*.node
  test-lancedb:
    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
    needs:
      - build-lancedb
    strategy:
      fail-fast: false
      matrix:
        settings:
          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
          # - host: windows-latest
          #   target: x86_64-pc-windows-msvc
          - host: macos-latest
            target: aarch64-apple-darwin
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-arm64
        node:
          - '20'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 11.1.1
      - name: Setup Node.js 24 for install
        uses: actions/setup-node@v4
        with:
          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
          # in October.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - name: Setup Node.js ${{ matrix.node }} for test
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node }}
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
          # run-id: 13982782871
      - name: List packages
        run: ls -R dist
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
        # Invoke jest directly because pnpm 11 itself requires Node 22+
        # while the matrix tests on older Node versions.
        run: npx jest --verbose
  publish:
    name: Publish
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    needs:
      - test-lancedb
    steps:
      - uses: actions/checkout@v4
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 11.1.1
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
          registry-url: "https://registry.npmjs.org"
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        name: Download arch-specific binaries
        with:
          pattern: lancedb-*
          path: nodejs/nodejs-artifacts
          merge-multiple: true
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - name: Display structure of downloaded files
        run: find dist && find nodejs-artifacts
      - name: Move artifacts
        run: pnpm exec napi artifacts -d nodejs-artifacts
      - name: List packages
        run: find npm
      - name: Publish
        env:
          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        run: |
          npm config set provenance true
          ARGS="--access public"
          if [[ $DRY_RUN == "true" ]]; then
            ARGS="$ARGS --dry-run"
          fi
          VERSION=$(node -p "require('./package.json').version")
          if [[ $VERSION == *-* ]]; then
            ARGS="$ARGS --tag preview"
          fi
          npm publish $ARGS
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [build-lancedb, test-lancedb, publish]
    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}