greptimedb/.github/workflows/release.yml

on:
  push:
    tags:
      - "v*.*.*"
  schedule:
    # At 00:00 on Monday.
    - cron: '0 0 * * 1'
  # Mannually trigger only builds binaries.
  workflow_dispatch:

name: Release

env:
  RUST_TOOLCHAIN: nightly-2023-02-26

  SCHEDULED_BUILD_VERSION_PREFIX: v0.2.0

  SCHEDULED_PERIOD: nightly

  CARGO_PROFILE: nightly

  # Controls whether to run tests, include unit-test, integration-test and sqlness.
  DISABLE_RUN_TESTS: false

jobs:
  build:
    name: Build binary
    strategy:
      matrix:
        # The file format is greptime-<os>-<arch>
        include:
          - arch: x86_64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-amd64
            continue-on-error: false
            opts: "-F servers/dashboard"
          - arch: aarch64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-arm64
            continue-on-error: false
            opts: "-F servers/dashboard"
          - arch: aarch64-apple-darwin
            os: macos-latest
            file: greptime-darwin-arm64
            continue-on-error: false
            opts: "-F servers/dashboard"
          - arch: x86_64-apple-darwin
            os: macos-latest
            file: greptime-darwin-amd64
            continue-on-error: false
            opts: "-F servers/dashboard"
          - arch: x86_64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-amd64-pyo3
            continue-on-error: false
            opts: "-F pyo3_backend,servers/dashboard"
          - arch: aarch64-unknown-linux-gnu
            os: ubuntu-2004-16-cores
            file: greptime-linux-arm64-pyo3
            continue-on-error: false
            opts: "-F pyo3_backend,servers/dashboard"
          - arch: aarch64-apple-darwin
            os: macos-latest
            file: greptime-darwin-arm64-pyo3
            continue-on-error: false
            opts: "-F pyo3_backend,servers/dashboard"
          - arch: x86_64-apple-darwin
            os: macos-latest
            file: greptime-darwin-amd64-pyo3
            continue-on-error: false
            opts: "-F pyo3_backend,servers/dashboard"
    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.continue-on-error }}
    if: github.repository == 'GreptimeTeam/greptimedb'
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Cache cargo assets
        id: cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ matrix.arch }}-build-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Install Protoc for linux
        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
        run: | # Make sure the protoc is >= 3.15
          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip
          unzip protoc-21.9-linux-x86_64.zip -d protoc
          sudo cp protoc/bin/protoc /usr/local/bin/
          sudo cp -r protoc/include/google /usr/local/include/

      - name: Install Protoc for macos
        if: contains(matrix.arch, 'darwin')
        run: |
          brew install protobuf

      - name: Install etcd for linux
        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
        run: |
          ETCD_VER=v3.5.7
          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
          mkdir -p /tmp/etcd-download
          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1
          rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

          sudo cp -a /tmp/etcd-download/etcd* /usr/local/bin/
          nohup etcd >/tmp/etcd.log 2>&1 &

      - name: Install etcd for macos
        if: contains(matrix.arch, 'darwin')
        run: |
          brew install etcd
          brew services start etcd

      - name: Install dependencies for linux
        if: contains(matrix.arch, 'linux') && endsWith(matrix.arch, '-gnu')
        run: |
          sudo apt-get -y update
          sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget

      # FIXME(zyy17): Should we specify the version of python when building binary for darwin?
      - name: Compile Python 3.10.10 from source for linux
        if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend')
        run: |
          sudo chmod +x ./docker/aarch64/compile-python.sh
          sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }}

      - name: Install rust toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
          targets: ${{ matrix.arch }}

      - name: Output package versions
        run: protoc --version ; cargo version ; rustc --version ; gcc --version ; g++ --version

      - name: Run tests
        if: env.DISABLE_RUN_TESTS == 'false'
        run: make unit-test integration-test sqlness-test

      - name: Run cargo build with pyo3 for aarch64-linux
        if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
        run: |
          # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable.
          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH

          export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64
          echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR"
          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip

          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}

      - name: Run cargo build with pyo3 for amd64-linux
        if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend')
        run: |
          export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64
          export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH
          export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH
          export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH

          echo "implementation=CPython" >> pyo3.config
          echo "version=3.10" >> pyo3.config
          echo "implementation=CPython" >> pyo3.config
          echo "shared=true" >> pyo3.config
          echo "abi3=true" >> pyo3.config
          echo "lib_name=python3.10" >> pyo3.config
          echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config
          echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config
          echo "pointer_width=64" >> pyo3.config
          echo "build_flags=" >> pyo3.config
          echo "suppress_build_script_link_lines=false" >> pyo3.config

          cat pyo3.config
          export PYO3_CONFIG_FILE=${PWD}/pyo3.config
          alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3
          alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip

          cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}

      - name: Run cargo build
        if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false
        run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }}

      - name: Calculate checksum and rename binary
        shell: bash
        run: |
          cd target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}
          chmod +x greptime
          tar -zcvf ${{ matrix.file }}.tgz greptime
          echo $(shasum -a 256 ${{ matrix.file }}.tgz | cut -f1 -d' ') > ${{ matrix.file }}.sha256sum

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.file }}
          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.tgz

      - name: Upload checksum of artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.file }}.sha256sum
          path: target/${{ matrix.arch }}/${{ env.CARGO_PROFILE }}/${{ matrix.file }}.sha256sum

  docker:
    name: Build docker image
    needs: [build]
    runs-on: ubuntu-latest
    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Login to Dockerhub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
        shell: bash
        if: github.event_name == 'schedule'
        run: |
          buildTime=`date "+%Y%m%d"`
          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV

      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
        shell: bash
        if: github.event_name != 'schedule'
        run: |
          VERSION=${{ github.ref_name }}
          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up buildx
        uses: docker/setup-buildx-action@v2

      - name: Download amd64 binary
        uses: actions/download-artifact@v3
        with:
          name: greptime-linux-amd64-pyo3
          path: amd64

      - name: Unzip the amd64 artifacts
        run: |
          tar xvf amd64/greptime-linux-amd64-pyo3.tgz -C amd64/ && rm amd64/greptime-linux-amd64-pyo3.tgz
          cp -r amd64 docker/ci

      - name: Download arm64 binary
        id: download-arm64
        uses: actions/download-artifact@v3
        with:
          name: greptime-linux-arm64-pyo3
          path: arm64

      - name: Unzip the arm64 artifacts
        id: unzip-arm64
        if: success() || steps.download-arm64.conclusion == 'success'
        run: |
          tar xvf arm64/greptime-linux-arm64-pyo3.tgz -C arm64/ && rm arm64/greptime-linux-arm64-pyo3.tgz
          cp -r arm64 docker/ci

      - name: Build and push all
        uses: docker/build-push-action@v3
        if: success() || steps.unzip-arm64.conclusion == 'success' # Build and push all platform if unzip-arm64 succeeds
        with:
          context: ./docker/ci/
          file: ./docker/ci/Dockerfile
          push: true
          platforms: linux/amd64,linux/arm64
          tags: |
            greptime/greptimedb:latest
            greptime/greptimedb:${{ env.IMAGE_TAG }}

      - name: Build and push amd64 only
        uses: docker/build-push-action@v3
        if: success() || steps.download-arm64.conclusion == 'failure' # Only build and push amd64 platform if download-arm64 fails
        with:
          context: ./docker/ci/
          file: ./docker/ci/Dockerfile
          push: true
          platforms: linux/amd64
          tags: |
            greptime/greptimedb:latest
            greptime/greptimedb:${{ env.IMAGE_TAG }}

  release:
    name: Release artifacts
    # Release artifacts only when all the artifacts are built successfully.
    needs: [build,docker]
    runs-on: ubuntu-latest
    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Download artifacts
        uses: actions/download-artifact@v3

      - name: Configure scheduled build version # the version would be ${SCHEDULED_BUILD_VERSION_PREFIX}-${SCHEDULED_PERIOD}-YYYYMMDD, like v0.2.0-nigthly-20230313.
        shell: bash
        if: github.event_name == 'schedule'
        run: |
          buildTime=`date "+%Y%m%d"`
          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-${{ env.SCHEDULED_PERIOD }}-$buildTime
          echo "SCHEDULED_BUILD_VERSION=${SCHEDULED_BUILD_VERSION}" >> $GITHUB_ENV

      # Only publish release when the release tag is like v1.0.0, v1.0.1, v1.0.2, etc.
      - name: Set whether it is the latest release
        run: |
          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            echo "prerelease=false" >> $GITHUB_ENV
            echo "makeLatest=true" >> $GITHUB_ENV
          else
            echo "prerelease=true" >> $GITHUB_ENV
            echo "makeLatest=false" >> $GITHUB_ENV
          fi

      - name: Create scheduled build git tag
        if: github.event_name == 'schedule'
        run: |
          git tag ${{ env.SCHEDULED_BUILD_VERSION }}

      - name: Publish scheduled release # configure the different release title and tags.
        uses: ncipollo/release-action@v1
        if: github.event_name == 'schedule'
        with:
          name: "Release ${{ env.SCHEDULED_BUILD_VERSION }}"
          prerelease: ${{ env.prerelease }}
          makeLatest: ${{ env.makeLatest }}
          tag: ${{ env.SCHEDULED_BUILD_VERSION }}
          generateReleaseNotes: true
          artifacts: |
            **/greptime-*

      - name: Publish release
        uses: ncipollo/release-action@v1
        if: github.event_name != 'schedule'
        with:
          name: "${{ github.ref_name }}"
          prerelease: ${{ env.prerelease }}
          makeLatest: ${{ env.makeLatest }}
          generateReleaseNotes: true
          artifacts: |
            **/greptime-*

  docker-push-acr:
    name: Push docker image to alibaba cloud container registry
    needs: [docker]
    runs-on: ubuntu-latest
    if: github.repository == 'GreptimeTeam/greptimedb' && github.event_name != 'workflow_dispatch'
    continue-on-error: true
    steps:
      - name: Checkout sources
        uses: actions/checkout@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to alibaba cloud container registry
        uses: docker/login-action@v2
        with:
          registry: registry.cn-hangzhou.aliyuncs.com
          username: ${{ secrets.ALICLOUD_USERNAME }}
          password: ${{ secrets.ALICLOUD_PASSWORD }}

      - name: Configure scheduled build image tag # the tag would be ${SCHEDULED_BUILD_VERSION_PREFIX}-YYYYMMDD-${SCHEDULED_PERIOD}
        shell: bash
        if: github.event_name == 'schedule'
        run: |
          buildTime=`date "+%Y%m%d"`
          SCHEDULED_BUILD_VERSION=${{ env.SCHEDULED_BUILD_VERSION_PREFIX }}-$buildTime-${{ env.SCHEDULED_PERIOD }}
          echo "IMAGE_TAG=${SCHEDULED_BUILD_VERSION:1}" >> $GITHUB_ENV

      - name: Configure tag # If the release tag is v0.1.0, then the image version tag will be 0.1.0.
        shell: bash
        if: github.event_name != 'schedule'
        run: |
          VERSION=${{ github.ref_name }}
          echo "IMAGE_TAG=${VERSION:1}" >> $GITHUB_ENV

      - name: Push image to alibaba cloud container registry # Use 'docker buildx imagetools create' to create a new image base on source image.
        run: |
          docker buildx imagetools create \
            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:latest \
            --tag registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:${{ env.IMAGE_TAG }} \
            greptime/greptimedb:${{ env.IMAGE_TAG }}