ci: retry failed nightly release on following weekdays (#9394)

Previously the nightly release was scheduled only on Mondays, so a
failed run left users without nightly builds for a whole week.

Now the schedule triggers every weekday at 00:00 UTC, but the release
only proceeds when the latest published nightly release is older than
NIGHTLY_RELEASE_MAX_AGE_DAYS (5) days. The check runs in
allocate-runners before any EC2 runner is allocated, and gates all
schedule-driven jobs (including the Slack notification, so a skipped
nightly is silent). Tag pushes and manual dispatches are unaffected.

Signed-off-by: Ning Sun <sunning@greptime.com>

.github/scripts/check-nightly-release.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Returns whether the current release run should proceed, 'true' or 'false'.
+#
+# The nightly schedule runs every weekday, but a scheduled release only proceeds
+# when the latest published nightly release is older than
+# NIGHTLY_RELEASE_MAX_AGE_DAYS days. This way, a failed nightly run is retried
+# by the next weekday schedule, instead of leaving a whole week without nightly
+# builds. Non-scheduled runs (tag pushes and manual dispatches) always proceed.
+#
+# Only the result is written to stdout, so that it can be used in GitHub Actions
+# outputs; all diagnostics go to stderr.
+#
+# You can run as following examples:
+#   GITHUB_EVENT_NAME=push ./check-nightly-release.sh
+#   GITHUB_EVENT_NAME=schedule GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh
+#   GITHUB_EVENT_NAME=schedule NIGHTLY_RELEASE_MAX_AGE_DAYS=5 GITHUB_REPOSITORY=GreptimeTeam/greptimedb GH_TOKEN=ghp_xxx ./check-nightly-release.sh
+if [ -z "$GITHUB_EVENT_NAME" ]; then
+    echo "GITHUB_EVENT_NAME is empty" >&2
+    exit 1
+fi
+
+if [ "$GITHUB_EVENT_NAME" != "schedule" ]; then
+    echo "Not a scheduled run, the nightly check is skipped." >&2
+    echo "true"
+    exit 0
+fi
+
+MAX_AGE_DAYS="${NIGHTLY_RELEASE_MAX_AGE_DAYS:-5}"
+if ! [[ "$MAX_AGE_DAYS" =~ ^[0-9]+$ ]]; then
+    echo "Invalid NIGHTLY_RELEASE_MAX_AGE_DAYS: ${MAX_AGE_DAYS}" >&2
+    exit 1
+fi
+
+SECONDS_PER_DAY=86400
+
+# The publish time of the latest published nightly release, like the tag
+# 'v0.2.0-nightly-20230313'. Fail open on GitHub API errors: proceeding with
+# the release is safer than skipping it.
+if ! latest_nightly_published_at="$(gh api "repos/${GITHUB_REPOSITORY}/releases?per_page=100" \
+    --jq '[.[] | select(.draft == false and (.tag_name | contains("nightly")))][0].published_at // empty')"; then
+    echo "::warning::Failed to query the latest nightly release from the GitHub API, will proceed with the nightly release." >&2
+    echo "true"
+    exit 0
+fi
+
+if [ -z "$latest_nightly_published_at" ]; then
+    echo "No published nightly release found, will proceed with the nightly release." >&2
+    echo "true"
+    exit 0
+fi
+
+published_at_epoch="$(date --date="$latest_nightly_published_at" +%s)"
+now_epoch="$(date +%s)"
+age_seconds=$(( now_epoch - published_at_epoch ))
+age_days=$(( age_seconds / SECONDS_PER_DAY ))
+
+echo "The latest nightly release was published at ${latest_nightly_published_at} (${age_days} days ago)." >&2
+
+if [ "$age_seconds" -gt $(( MAX_AGE_DAYS * SECONDS_PER_DAY )) ]; then
+    echo "The latest nightly release is older than ${MAX_AGE_DAYS} days, will proceed with the nightly release." >&2
+    echo "true"
+else
+    echo "The latest nightly release is not older than ${MAX_AGE_DAYS} days, will skip the nightly release." >&2
+    echo "false"
+fi

.github/workflows/release.yml

@@ -4,15 +4,20 @@ name: Release
 # 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event.
 # 2. The scheduled release(the version will be '<base-version>-nightly-YYYYMMDD', any pre-release
 #    extension in the next release version like '-alpha.1' is stripped, so 'nightly' is the extension):
-#    the release workflow will be triggered by the schedule event.
+#    the release workflow will be triggered by the schedule event. The schedule runs every weekday,
+#    but the release only proceeds when the latest published nightly release is older than
+#    NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday
+#    schedule instead of leaving a whole week without nightly builds.
 # A failed tag push release can be manually dispatched on the same tag to recover it without changing the formal release version.
 on:
   push:
     tags:
       - "v*.*.*"
   schedule:
-    # At 00:00 on Monday.
-    - cron: '0 0 * * 1'
+    # At 00:00 on every weekday (Monday to Friday). The scheduled nightly release only
+    # proceeds when the latest published nightly release is older than
+    # NIGHTLY_RELEASE_MAX_AGE_DAYS days, see the 'Check nightly release necessity' step.
+    - cron: '0 0 * * 1-5'
   workflow_dispatch: # Allows you to run this workflow manually.
     # Notes: The GitHub Actions ONLY support 10 inputs; this final input is the tenth.
     inputs:
@@ -100,6 +105,10 @@ env:
   # the pre-release extension of the next release version is stripped, e.g. v0.2.0-alpha.1 -> v0.2.0-nightly-20230313.
   NIGHTLY_RELEASE_PREFIX: nightly
 
+  # The maximum age (in days) of the latest published nightly release before a scheduled
+  # nightly release may run again; see the 'Check nightly release necessity' step.
+  NIGHTLY_RELEASE_MAX_AGE_DAYS: 5
+
 jobs:
   allocate-runners:
     name: Allocate runners
@@ -123,6 +132,10 @@ jobs:
       # The 'is-current-version-latest' determines whether to update 'latest' Docker tags and downstream repositories.
       is-current-version-latest: ${{ steps.check-version.outputs.is-current-version-latest }}
       is-current-version-stable: ${{ steps.check-version.outputs.is-current-version-stable }}
+
+      # Whether a scheduled nightly release should run: it only runs when the latest published
+      # nightly release is older than NIGHTLY_RELEASE_MAX_AGE_DAYS days. Always 'true' for other events.
+      nightly-required: ${{ steps.check-nightly.outputs.nightly-required }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -135,6 +148,19 @@ jobs:
         run: |
           ./scripts/check-builder-rust-version.sh
 
+      # A scheduled run only proceeds when the latest published nightly release is older than
+      # NIGHTLY_RELEASE_MAX_AGE_DAYS days, so a failed nightly run is retried by the next weekday
+      # schedule. The check runs before the runners are allocated to avoid wasting EC2 resources.
+      - name: Check nightly release necessity
+        id: check-nightly
+        run: |
+          echo "nightly-required=$(./.github/scripts/check-nightly-release.sh)" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          NIGHTLY_RELEASE_MAX_AGE_DAYS: ${{ env.NIGHTLY_RELEASE_MAX_AGE_DAYS }}
+
       # The create-version will create a global variable named 'version' in the global workflows.
       # - If it's a tag push release or manual dispatch on a tag with REUSE_EXISTING_RELEASE_TAG=true, the version is the tag name(${{ github.ref_name }});
       # - If it's a scheduled release, the version is '<base-version>-nightly-$buildTime', like v0.2.0-nightly-20230313 (pre-release extensions like '-alpha.1' are stripped);
@@ -156,7 +182,7 @@ jobs:
           ./.github/scripts/check-version.sh "${{ steps.create-version.outputs.version }}"
 
       - name: Allocate linux-amd64 runner
-        if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+        if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
         uses: ./.github/actions/start-runner
         id: start-linux-amd64-runner
         with:
@@ -170,7 +196,7 @@ jobs:
           subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
 
       - name: Allocate linux-arm64 runner
-        if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+        if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && steps.check-nightly.outputs.nightly-required == 'true' }}
         uses: ./.github/actions/start-runner
         id: start-linux-arm64-runner
         with:
@@ -185,7 +211,7 @@ jobs:
 
   build-linux-amd64-artifacts:
     name: Build linux-amd64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -211,7 +237,7 @@ jobs:
 
   build-linux-arm64-artifacts:
     name: Build linux-arm64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -238,7 +264,7 @@ jobs:
 
   build-linux-riscv64-artifacts:
     name: Build linux-riscv64 artifacts
-    if: ${{ inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_linux_artifacts || inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -302,7 +328,7 @@ jobs:
     needs: [
       allocate-runners,
     ]
-    if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -340,7 +366,7 @@ jobs:
     needs: [
       allocate-runners,
     ]
-    if: ${{ inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.build_windows_artifacts || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     steps:
       - run: git config --global core.autocrlf false
 
@@ -365,7 +391,7 @@ jobs:
 
   prepare-release-validation:
     name: Prepare release validation
-    if: ${{ inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.publish_github_release || inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [
       allocate-runners,
     ]
@@ -528,6 +554,7 @@ jobs:
     if: |
       always() &&
       (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') &&
       needs.allocate-runners.result == 'success' &&
       needs.build-linux-amd64-artifacts.result == 'success' &&
       needs.build-linux-arm64-artifacts.result == 'success' &&
@@ -571,7 +598,7 @@ jobs:
 
   release-cn-artifacts:
     name: Release artifacts to CN region
-    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
+    if: ${{ (inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule') && (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') }}
     needs: [ # The job have to wait for all the artifacts are built.
       allocate-runners,
       build-linux-amd64-artifacts,
@@ -618,6 +645,7 @@ jobs:
     if: |
       always() &&
       (inputs.publish_github_release || github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.event_name != 'schedule' || needs.allocate-runners.outputs.nightly-required == 'true') &&
       needs.allocate-runners.result == 'success' &&
       needs.prepare-release-validation.result == 'success' &&
       (needs.compat-release.result == 'success' ||
@@ -726,7 +754,7 @@ jobs:
 
   bump-downstream-repo-versions:
     name: Bump downstream repo versions
-    if: ${{ github.event_name == 'schedule' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }}
+    if: ${{ (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true') || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_type == 'tag' && !contains(github.ref_name, 'nightly')) && (needs.allocate-runners.outputs.is-current-version-stable == 'true' && needs.allocate-runners.outputs.is-current-version-latest == 'true') }}
     needs: [allocate-runners, publish-github-release]
     runs-on: ubuntu-latest
     # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
@@ -793,9 +821,10 @@ jobs:
           ./.github/scripts/update-homebrew-greptme-version.sh
 
   notification:
-    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || github.event_name == 'schedule') && always() }}
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'push' || (github.event_name == 'schedule' && needs.allocate-runners.outputs.nightly-required == 'true')) && always() }}
     name: Send notification to Greptime team
     needs: [
+      allocate-runners,
       release-images-to-dockerhub,
       build-macos-artifacts,
       build-windows-artifacts,