diff --git a/.github/scripts/check-nightly-release.sh b/.github/scripts/check-nightly-release.sh new file mode 100755 index 00000000000..82eb910d6fd --- /dev/null +++ b/.github/scripts/check-nightly-release.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Returns whether the current release run should proceed, 'true' or 'false'. +# +# The nightly schedule runs every weekday, but a scheduled release only proceeds +# when the latest published nightly release is older than +# NIGHTLY_RELEASE_MAX_AGE_DAYS days. This way, a failed nightly run is retried +# by the next weekday schedule, instead of leaving a whole week without nightly +# builds. Non-scheduled runs (tag pushes and manual dispatches) always proceed. +# +# Only the result is written to stdout, so that it can be used in GitHub Actions +# outputs; all diagnostics go to stderr. +# +# You can run as following examples: +# GITHUB_EVENT_NAME=push ./check-nightly-release.sh +# GITHUB_EVENT_NAME=schedule GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh +# GITHUB_EVENT_NAME=schedule NIGHTLY_RELEASE_MAX_AGE_DAYS=5 GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh +if [ -z "$GITHUB_EVENT_NAME" ]; then + echo "GITHUB_EVENT_NAME is empty" >&2 + exit 1 +fi + +if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then + echo "Not a scheduled run, the nightly check is skipped." >&2 + echo "true" + exit 0 +fi + +MAX_AGE_DAYS="${NIGHTLY_RELEASE_MAX_AGE_DAYS:-5}" +if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then + echo "Invalid NIGHTLY_RELEASE_MAX_AGE_DAYS: ${MAX_AGE_DAYS}" >&2 + exit 1 +fi + +SECONDS_PER_DAY=86400 + +# The publish time of the latest published nightly release, like the tag +# 'v0.2.0-nightly-20230313'. Fail open on GitHub API errors: proceeding with +# the release is safer than skipping it. +if ! latest_nightly_published_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \ + --jq '[.[] | select(.draft == false and (.tag_name | contains("nightly")))][0].published_at // empty')"; then + echo "::warning::Failed to query the latest nightly release from the GitHub API, will proceed with the nightly release." >&2 + echo "true" + exit 0 +fi + +if [ -z "$latest_nightly_published_at" ]; then + echo "No published nightly release found, will proceed with the nightly release." >&2 + echo "true" + exit 0 +fi + +published_at_epoch="$(date --date="$latest_nightly_published_at" +%s)" +now_epoch="$(date +%s)" +age_seconds=$(( now_epoch - published_at_epoch )) +age_days=$(( age_seconds / SECONDS_PER_DAY )) + +echo "The latest nightly release was published at ${latest_nightly_published_at} (${age_days} days ago)." >&2 + +if [ "$age_seconds" -gt $(( MAX_AGE_DAYS * SECONDS_PER_DAY )) ]; then + echo "The latest nightly release is older than ${MAX_AGE_DAYS} days, will proceed with the nightly release." >&2 + echo "true" +else + echo "The latest nightly release is not older than ${MAX_AGE_DAYS} days, will skip the nightly release." >&2 + echo "false" +fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c0c36c895e..5a689910aee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,15 +4,20 @@ name: Release # 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event. # 2. The scheduled release(the version will be '-nightly-YYYYMMDD', any pre-release # extension in the next release version like '-alpha.1' is stripped, so 'nightly' is the extension): -# the release workflow will be triggered by the schedule event. +# the release workflow will be triggered by the schedule event. The schedule runs every weekday, +# but the release only proceeds when the latest published nightly release is older than +# NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday +# schedule instead of leaving a whole week without nightly builds. # A failed tag push release can be manually dispatched on the same tag to recover it without changing the formal release version. on: push: tags: - "v*.*.*" schedule: - # At 00:00 on Monday. - - cron: '0 0 * * 1' + # At 00:00 on every weekday (Monday to Friday). The scheduled nightly release only + # proceeds when the latest published nightly release is older than + # NIGHTLY_RELEASE_MAX_AGE_DAYS days, see the 'Check nightly release necessity' step. + - cron: '0 0 * * 1-5' workflow_dispatch: # Allows you to run this workflow manually. # Notes: The GitHub Actions ONLY support 10 inputs; this final input is the tenth. inputs: @@ -100,6 +105,10 @@ env: # the pre-release extension of the next release version is stripped, e.g. v0.2.0-alpha.1 -> v0.2.0-nightly-20230313. NIGHTLY_RELEASE_PREFIX: nightly + # The maximum age (in days) of the latest published nightly release before a scheduled + # nightly release may run again; see the 'Check nightly release necessity' step. + NIGHTLY_RELEASE_MAX_AGE_DAYS: 5 + jobs: allocate-runners: name: Allocate runners @@ -123,6 +132,10 @@ jobs: # The 'is-current-version-latest' determines whether to update 'latest' Docker tags and downstream repositories. is-current-version-latest: ${{ steps.check-version.outputs.is-current-version-latest }} is-current-version-stable: ${{ steps.check-version.outputs.is-current-version-stable }} + + # Whether a scheduled nightly release should run: it only runs when the latest published + # nightly release is older than NIGHTLY_RELEASE_MAX_AGE_DAYS days. Always 'true' for other events. + nightly-required: ${{ steps.check-nightly.outputs.nightly-required }} steps: - name: Checkout uses: actions/checkout@v4 @@ -135,6 +148,19 @@ jobs: run: | ./scripts/check-builder-rust-version.sh + # A scheduled run only proceeds when the latest published nightly release is older than + # NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday + # schedule. The check runs before the runners are allocated to avoid wasting EC2 resources. + - name: Check nightly release necessity + id: check-nightly + run: | + echo "nightly-required=$(./.github/scripts/check-nightly-release.sh)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REPOSITORY: ${{ github.repository }} + NIGHTLY_RELEASE_MAX_AGE_DAYS: ${{ env.NIGHTLY_RELEASE_MAX_AGE_DAYS }} + # The create-version will create a global variable named 'version' in the global workflows. # - If it's a tag push release or manual dispatch on a tag with REUSE_EXISTING_RELEASE_TAG=true, the version is the tag name(${{ github.ref_name }}); # - If it's a scheduled release, the version is '-nightly-$buildTime', like v0.2.0-nightly-20230313 (pre-release extensions like '-alpha.1' are stripped); @@ -156,7 +182,7 @@ jobs: ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}" - name: Allocate linux-amd64 runner - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }} uses: ./.github/actions/start-runner id: start-linux-amd64-runner with: @@ -170,7 +196,7 @@ jobs: subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} - name: Allocate linux-arm64 runner - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }} uses: ./.github/actions/start-runner id: start-linux-arm64-runner with: @@ -185,7 +211,7 @@ jobs: build-linux-amd64-artifacts: name: Build linux-amd64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -211,7 +237,7 @@ jobs: build-linux-arm64-artifacts: name: Build linux-arm64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -238,7 +264,7 @@ jobs: build-linux-riscv64-artifacts: name: Build linux-riscv64 artifacts - if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -302,7 +328,7 @@ jobs: needs: [ allocate-runners, ] - if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} steps: - uses: actions/checkout@v4 with: @@ -340,7 +366,7 @@ jobs: needs: [ allocate-runners, ] - if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} steps: - run: git config --global core.autocrlf false @@ -365,7 +391,7 @@ jobs: prepare-release-validation: name: Prepare release validation - if: ${{ inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ allocate-runners, ] @@ -528,6 +554,7 @@ jobs: if: | always() && (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && + (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') && needs.allocate-runners.result == 'success' && needs.build-linux-amd64-artifacts.result == 'success' && needs.build-linux-arm64-artifacts.result == 'success' && @@ -571,7 +598,7 @@ jobs: release-cn-artifacts: name: Release artifacts to CN region - if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }} + if: ${{ (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }} needs: [ # The job have to wait for all the artifacts are built. allocate-runners, build-linux-amd64-artifacts, @@ -618,6 +645,7 @@ jobs: if: | always() && (inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule') && + (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') && needs.allocate-runners.result == 'success' && needs.prepare-release-validation.result == 'success' && (needs.compat-release.result == 'success' || @@ -726,7 +754,7 @@ jobs: bump-downstream-repo-versions: name: Bump downstream repo versions - if: ${{ github.event_name == 'schedule' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }} + if: ${{ (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true') || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }} needs: [allocate-runners, publish-github-release] runs-on: ubuntu-latest # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs @@ -793,9 +821,10 @@ jobs: ./.github/scripts/update-homebrew-greptme-version.sh notification: - if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }} + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true')) && always() }} name: Send notification to Greptime team needs: [ + allocate-runners, release-images-to-dockerhub, build-macos-artifacts, build-windows-artifacts,