lancedb/.github/workflows/npm-publish.yml

name: NPM Publish

env:
  MACOSX_DEPLOYMENT_TARGET: '10.13'
  CARGO_INCREMENTAL: '0'

permissions:
  contents: write
  id-token: write

on:
  push:
    tags:
      - "v*"
  # The cross-compiled targets (musl especially) break from toolchain and
  # dependency changes that nothing else in CI catches, and discovering that
  # mid-release is expensive. A nightly run keeps that signal while dropping
  # the full 8-target release matrix from all ~90 pushes to main each month.
  # `report-failure` files an issue when a nightly breaks.
  schedule:
    - cron: "0 8 * * *"
  workflow_dispatch:
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/npm-publish.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-lancedb:
    strategy:
      fail-fast: false
      matrix:
        settings:
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
            pre_build: |-
              brew install protobuf
              # Fat LTO (the workspace default in .cargo/config.toml) is
              # single-threaded and is the peak-memory step of the build. On
              # this runner it accounted for ~111 of the job's ~113 minutes,
              # making it the critical path of the entire publish pipeline.
              # ThinLTO parallelizes it across the runner's cores, for a few
              # percent of runtime performance.
              export CARGO_PROFILE_RELEASE_LTO=thin
              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-pc-windows-msvc
            host: windows-2025
            features: ","
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
              # See the ThinLTO note on aarch64-apple-darwin above. Keeping
              # peak memory down is also what lets this run on the standard
              # 4-core runner: the 8-core larger runner was only needed to
              # stop fat LTO from OOMing rustc-LLVM.
              export CARGO_PROFILE_RELEASE_LTO=thin
              export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: aarch64-pc-windows-msvc
            host: windows-2025
            features: ","
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
                # See the ThinLTO note on aarch64-apple-darwin above.
                export CARGO_PROFILE_RELEASE_LTO=thin
                export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              # The base image (manylinux2014-cross) sets TARGET_CC to the old
              # GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
              # so it picks up GCC even though the napi-rs image sets CC=clang.
              # Override to use the image's clang-18 which supports -fuse-ld=lld.
              export TARGET_CC=clang TARGET_CXX=clang++
          - target: x86_64-unknown-linux-musl
            # This one seems to need some extra memory
            host: ubuntu-2404-8x-x64
            features: fp16kernels
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler pkg-config &&
              rustup target add x86_64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              export TARGET_CC=clang TARGET_CXX=clang++ &&
              # The manylinux2014 sysroot has glibc 2.17 headers which lack
              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
              rustup target add aarch64-unknown-linux-gnu
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            features: ","
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler &&
              rustup target add aarch64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v6
      - name: Setup pnpm
        uses: pnpm/action-setup@v6
        with:
          version: 11.1.1
      - name: Setup node
        uses: actions/setup-node@v6
        with:
          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
          # in October.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
      - name: Install
        uses: dtolnay/rust-toolchain@stable
        if: ${{ !matrix.settings.docker }}
        with:
          toolchain: stable
          targets: ${{ matrix.settings.target }}
      # These builds were entirely uncached: the old key was static, so
      # `actions/cache` (which only writes on a miss) could never refresh it,
      # and the multi-GB whole-`target/` copy it tried to store never fit the
      # repo's cache budget, so no entry was ever saved. rust-cache prunes
      # `target/` to dependency artifacts and keys on Cargo.lock plus the rustc
      # version, which both fixes the key and keeps entries a sane size.
      #
      # This caches dependency *compilation* only. The LTO link of the cdylib
      # re-runs regardless, since the local crate changes every time, so the
      # win is larger on the non-LTO jobs than here.
      - name: Cache cargo (native builds)
        uses: Swatinem/rust-cache@v2
        if: ${{ !matrix.settings.docker }}
        with:
          # The release profile and per-target dirs differ from what the test
          # workflows cache, so these need to be separate entries.
          key: release-${{ matrix.settings.target }}
          # Only the nightly run on main writes, so tag and PR runs restore a
          # warm entry without every dependabot PR writing its own (which would
          # be unreadable elsewhere anyway, since GitHub scopes caches to the
          # creating ref). The nightly cadence also keeps entries inside
          # GitHub's 7-day eviction window, which a tag-only trigger would not.
          save-if: ${{ github.ref == 'refs/heads/main' }}
      # Docker builds can use rust-cache too. `target/` already lives on the
      # host because the whole workspace is bind-mounted into the container, and
      # rust-cache's prune and save run host-side, so they can manage it -- which
      # is what keeps the entry to dependency artifacts rather than a multi-GB
      # copy of everything.
      #
      # Two differences from the native builds. The container's CARGO_HOME is
      # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
      # has to be cached explicitly. And the key is derived from the *host* rustc
      # version, which is not the compiler that produced these artifacts; that is
      # safe because cargo fingerprints the real compiler and rebuilds on a
      # mismatch, it just means a base-image toolchain bump costs one cold build
      # instead of invalidating the key.
      - name: Cache cargo (docker builds)
        uses: Swatinem/rust-cache@v2
        if: ${{ matrix.settings.docker }}
        with:
          key: docker-${{ matrix.settings.target }}
          cache-directories: .cargo-cache
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - name: Install Zig
        uses: mlugg/setup-zig@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          version: 0.14.1
      - name: Install cargo-zigbuild
        uses: taiki-e/install-action@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          tool: cargo-zigbuild
      - name: Build in docker
        uses: addnab/docker-run-action@v3
        if: ${{ matrix.settings.docker }}
        with:
          image: ${{ matrix.settings.docker }}
          # All three mounts must live under `.cargo-cache`, which is what the
          # cache step above saves. Previously the registry mounts pointed at
          # `.cargo/...`, a path nothing cached, so the container re-downloaded
          # the whole crate registry on every run.
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
            npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              --output-dir dist/
      # The container runs as root (`--user 0:0`), so everything it wrote to the
      # mounted cache dirs is root-owned. rust-cache's post step runs as the
      # runner user and has to both read these and delete from them while
      # pruning, so hand them back before it runs.
      - name: Take ownership of docker build output
        if: ${{ matrix.settings.docker }}
        run: |
          sudo chown -R "$(id -u):$(id -g)" \
            "${{ github.workspace }}/.cargo-cache" \
            "${{ github.workspace }}/target"
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
          npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              $EXTRA_ARGS \
              --output-dir dist/
        if: ${{ !matrix.settings.docker }}
        shell: bash
      # The standard Windows runners have ~14 GB free, and a release `target/`
      # for this workspace is a large fraction of that. Report the remaining
      # headroom so a build that only just fits is visible before a dependency
      # bump turns it into a failed release. `always()` so the numbers are
      # still there when the build is what ran out of space.
      - name: Report disk headroom
        if: always()
        run: df -h
        shell: bash
      - name: Upload artifact
        uses: actions/upload-artifact@v7
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/*.node
          if-no-files-found: error
      # The generic files are the same in all distros so we just pick
      # one to do the upload.
      - name: Make generic artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        run: pnpm tsc
      - name: Upload Generic Artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        uses: actions/upload-artifact@v7
        with:
          name: nodejs-dist
          path: |
            nodejs/dist/*
            !nodejs/dist/*.node
  test-lancedb:
    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
    needs:
      - build-lancedb
    strategy:
      fail-fast: false
      matrix:
        settings:
          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
          # - host: windows-latest
          #   target: x86_64-pc-windows-msvc
          - host: macos-latest
            target: aarch64-apple-darwin
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-arm64
        node:
          - '20'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v6
      - name: Setup pnpm
        uses: pnpm/action-setup@v6
        with:
          version: 11.1.1
      - name: Setup Node.js 24 for install
        uses: actions/setup-node@v6
        with:
          # pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
          # in October.
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - name: Setup Node.js ${{ matrix.node }} for test
        uses: actions/setup-node@v6
        with:
          node-version: ${{ matrix.node }}
      - name: Download artifacts
        uses: actions/download-artifact@v8
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v8
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
          # run-id: 13982782871
      - name: List packages
        run: ls -R dist
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
        # Invoke jest directly because pnpm 11 itself requires Node 22+
        # while the matrix tests on older Node versions.
        run: npx jest --verbose
  publish:
    name: Publish
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    needs:
      - test-lancedb
    steps:
      - uses: actions/checkout@v6
      - name: Setup pnpm
        uses: pnpm/action-setup@v6
        with:
          version: 11.1.1
      - name: Setup node
        uses: actions/setup-node@v6
        with:
          node-version: 24
          cache: pnpm
          cache-dependency-path: nodejs/pnpm-lock.yaml
          registry-url: "https://registry.npmjs.org"
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
      - uses: actions/download-artifact@v8
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v8
        name: Download arch-specific binaries
        with:
          pattern: lancedb-*
          path: nodejs/nodejs-artifacts
          merge-multiple: true
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - name: Display structure of downloaded files
        run: find dist && find nodejs-artifacts
      - name: Move artifacts
        run: pnpm exec napi artifacts -d nodejs-artifacts
      - name: List packages
        run: find npm
      - name: Publish
        env:
          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        run: |
          npm config set provenance true
          ARGS="--access public"
          if [[ $DRY_RUN == "true" ]]; then
            ARGS="$ARGS --dry-run"
          fi
          VERSION=$(node -p "require('./package.json').version")
          if [[ $VERSION == *-* ]]; then
            ARGS="$ARGS --tag preview"
          fi
          npm publish $ARGS
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [build-lancedb, test-lancedb, publish]
    # Nightly runs are the only thing watching the cross-compiled targets now,
    # so they have to report failures too or the signal is silently lost.
    if: always() && failure() && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'schedule')
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v6
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}