name: NPM Publish

env:
  MACOSX_DEPLOYMENT_TARGET: '10.13'
  CARGO_INCREMENTAL: '0'

permissions:
  contents: write
  id-token: write

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/npm-publish.yml
      - Cargo.toml # Change in dependency frequently breaks builds

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  gh-release:
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Release Notes
        id: release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.release_notes.outputs.changelog }}

  build-lancedb:
    strategy:
      fail-fast: false
      matrix:
        settings:
          - target: x86_64-apple-darwin
            host: macos-latest
            features: ","
            pre_build: |-
                brew install protobuf
                rustup target add x86_64-apple-darwin
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
            pre_build: brew install protobuf
          - target: x86_64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
          - target: aarch64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config
          - target: x86_64-unknown-linux-musl
            # This one seems to need some extra memory
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
            features: fp16kernels
            pre_build: |-
              set -e &&
              apk add protobuf-dev curl &&
              ln -s /usr/lib/gcc/x86_64-alpine-linux-musl/14.2.0/crtbeginS.o /usr/lib/crtbeginS.o &&
              ln -s /usr/lib/libgcc_s.so /usr/lib/libgcc.so &&
              CC=gcc &&
              CXX=g++
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              # https://github.com/aws/aws-lc-rs/issues/737#issuecomment-2725918627
              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/4.8.5/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o &&
              ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/gcc &&
              rustup target add aarch64-unknown-linux-gnu
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/alpine.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
            features: ","
            pre_build: |-
              set -e &&
              apk add protobuf-dev &&
              rustup target add aarch64-unknown-linux-musl &&
              export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc &&
              export CXX_aarch64_unknown_linux_musl=aarch64-linux-musl-g++
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        if: ${{ !matrix.settings.docker }}
        with:
          node-version: 20
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
      - name: Install
        uses: dtolnay/rust-toolchain@stable
        if: ${{ !matrix.settings.docker }}
        with:
          toolchain: stable
          targets: ${{ matrix.settings.target }}
      - name: Cache cargo
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
      - name: Setup toolchain
        run: ${{ matrix.settings.setup }}
        if: ${{ matrix.settings.setup }}
        shell: bash
      - name: Install dependencies
        run: npm ci
      - name: Build in docker
        uses: addnab/docker-run-action@v3
        if: ${{ matrix.settings.docker }}
        with:
          image: ${{ matrix.settings.docker }}
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
            npx napi build --platform  --release --no-const-enum \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              dist/
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
          npx napi build --platform  --release --no-const-enum \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              $EXTRA_ARGS \
              dist/
        if: ${{ !matrix.settings.docker }}
        shell: bash
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/*.node
          if-no-files-found: error
      # The generic files are the same in all distros so we just pick
      # one to do the upload.
      - name: Make generic artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        run: npm run tsc
      - name: Upload Generic Artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-dist
          path: |
            nodejs/dist/*
            !nodejs/dist/*.node
  test-lancedb:
    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
    needs:
      - build-lancedb
    strategy:
      fail-fast: false
      matrix:
        settings:
          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
          # - host: windows-latest
          #   target: x86_64-pc-windows-msvc
          - host: macos-latest
            target: aarch64-apple-darwin
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
          - target: aarch64-unknown-linux-gnu
            host: buildjet-16vcpu-ubuntu-2204-arm
        node:
          - '20'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node }}
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
          # run-id: 13982782871
      - name: List packages
        run: ls -R dist
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
        run: npm test
  publish:
    name: Publish
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    needs:
      - test-lancedb
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
          registry-url: "https://registry.npmjs.org"
      - name: Install dependencies
        run: npm ci
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        name: Download arch-specific binaries
        with:
          pattern: lancedb-*
          path: nodejs/nodejs-artifacts
          merge-multiple: true
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - name: Display structure of downloaded files
        run: find dist && find nodejs-artifacts
      - name: Move artifacts
        run: npx napi artifacts -d nodejs-artifacts
      - name: List packages
        run: find npm
      - name: Publish
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        run: |
          ARGS="--access public"
          if [[ $DRY_RUN == "true" ]]; then
            ARGS="$ARGS --dry-run"
          fi
          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
            ARGS="$ARGS --tag preview"
          fi
          npm publish $ARGS


  # ----------------------------------------------------------------------------
  # vectordb release (legacy)
  # ----------------------------------------------------------------------------
  # TODO: delete this when we drop vectordb
  node:
    name: vectordb Typescript
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: node
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - uses: actions/setup-node@v3
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: node/package-lock.json
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build
        run: |
          npm ci
          npm run tsc
          npm pack
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-package
          path: |
            node/vectordb-*.tgz

  node-macos:
    name: vectordb ${{ matrix.config.arch }}
    strategy:
      matrix:
        config:
          - arch: x86_64-apple-darwin
            runner: macos-13
          - arch: aarch64-apple-darwin
            # xlarge is implicitly arm64.
            runner: macos-14
    runs-on: ${{ matrix.config.runner }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install system dependencies
        run: brew install protobuf
      - name: Install npm dependencies
        run: |
          cd node
          npm ci
      - name: Build MacOS native node modules
        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
      - name: Upload Darwin Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-darwin-${{ matrix.config.arch }}
          path: |
            node/dist/lancedb-vectordb-darwin*.tgz

  node-linux-gnu:
    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
    runs-on: ${{ matrix.config.runner }}
    strategy:
      fail-fast: false
      matrix:
        config:
          - arch: x86_64
            runner: ubuntu-latest
          - arch: aarch64
            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
            runner: warp-ubuntu-latest-arm64-4x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      # To avoid OOM errors on ARM, we create a swap file.
      - name: Configure aarch64 build
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
          free -h
          sudo fallocate -l 16G /swapfile
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
          # print info
          swapon --show
          free -h
      - name: Build Linux Artifacts
        run: |
          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-linux-${{ matrix.config.arch }}-gnu
          path: |
            node/dist/lancedb-vectordb-linux*.tgz

  node-windows:
    name: vectordb ${{ matrix.target }}
    runs-on: windows-2022
    strategy:
      fail-fast: false
      matrix:
        target: [x86_64-pc-windows-msvc]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Protoc v21.12
        working-directory: C:\
        run: |
          New-Item -Path 'C:\protoc' -ItemType Directory
          Set-Location C:\protoc
          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
          7z x protoc.zip
          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
        shell: powershell
      - name: Install npm dependencies
        run: |
          cd node
          npm ci
      - name: Build Windows native node modules
        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
      - name: Upload Windows Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-windows
          path: |
            node/dist/lancedb-vectordb-win32*.tgz

  release:
    name: vectordb NPM Publish
    needs: [node, node-macos, node-linux-gnu, node-windows]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: node-*
      - name: Display structure of downloaded files
        run: ls -R
      - uses: actions/setup-node@v3
        with:
          node-version: 20
          registry-url: "https://registry.npmjs.org"
      - name: Publish to NPM
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
        run: |
          # Tag beta as "preview" instead of default "latest". See lancedb
          # npm publish step for more info.
          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
            PUBLISH_ARGS="--tag preview"
          fi

          mv */*.tgz .
          for filename in *.tgz; do
            npm publish $PUBLISH_ARGS $filename
          done
      - name: Deprecate
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
        # We need to deprecate the old package to avoid confusion.
        # Each time we publish a new version, it gets undeprecated.
        run: npm deprecate vectordb "Use @lancedb/lancedb instead."
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
      - name: Update package-lock.json
        run: |
          git config user.name 'Lance Release'
          git config user.email 'lance-dev@lancedb.com'
          bash ci/update_lockfiles.sh
      - name: Push new commit
        uses: ad-m/github-push-action@master
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
          branch: main
      - name: Notify Slack Action
        uses: ravsamhq/notify-slack-action@2.3.0
        if: ${{ always() }}
        with:
          status: ${{ job.status }}
          notify_when: "failure"
          notification_title: "{workflow} is failing"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}