From 1e39eab06bf84963f1fc42386ffd9bb18b846dd4 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Sat, 14 Nov 2026 22:16:28 +0000 Subject: [PATCH] fix(ci): bound nightly kind deployment and restore inotify limits (#10423) * fix(ci): bound nightly kind deployment and restore inotify limits Signed-off-by: evenyag * fix(ci): use shell timeouts in composite actions Signed-off-by: evenyag --------- Signed-off-by: evenyag --- .github/actions/deploy-greptimedb/action.yml | 9 +++++- .github/actions/sqlness-test/action.yml | 14 +++++++-- .github/scripts/deploy-greptimedb.sh | 30 +++++--------------- .github/workflows/nightly-ci.yml | 3 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/actions/deploy-greptimedb/action.yml b/.github/actions/deploy-greptimedb/action.yml index b18830ef8d7..634cae85dcd 100644 --- a/.github/actions/deploy-greptimedb/action.yml +++ b/.github/actions/deploy-greptimedb/action.yml @@ -19,6 +19,12 @@ inputs: runs: using: composite steps: + # Ubuntu 26.04 does not apply the runner's /etc/sysctl.conf at boot. + # https://github.com/actions/runner-images/issues/15108 + - name: Configure inotify limits for kind + shell: bash + run: sudo sysctl -w fs.inotify.max_user_instances=1280 fs.inotify.max_user_watches=655360 + - name: Deploy GreptimeDB by Helm shell: bash env: @@ -28,4 +34,5 @@ runs: AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} run: | - ./.github/scripts/deploy-greptimedb.sh + # Composite steps do not support timeout-minutes. + timeout --kill-after=30s 30m ./.github/scripts/deploy-greptimedb.sh diff --git a/.github/actions/sqlness-test/action.yml b/.github/actions/sqlness-test/action.yml index 004f2b40b16..255a175f7a0 100644 --- a/.github/actions/sqlness-test/action.yml +++ b/.github/actions/sqlness-test/action.yml @@ -58,11 +58,19 @@ runs: run: | aws s3 rm s3://${{ inputs.aws-ci-test-bucket }}/${{ inputs.data-root }} --recursive - name: Export kind logs - if: failure() + if: failure() || cancelled() shell: bash - run: kind export logs /tmp/kind + run: | + # Bound the entire collection while leaving the upload step available. + timeout --kill-after=30s 5m bash -e -o pipefail <<'EOF' + mkdir -p /tmp/kind + kubectl --request-timeout=30s get pods -A -o wide > /tmp/kind/pods.txt 2>&1 || true + kubectl --request-timeout=30s get events -A --sort-by=.metadata.creationTimestamp > /tmp/kind/events.txt 2>&1 || true + kubectl --request-timeout=30s describe pods -A > /tmp/kind/pod-descriptions.txt 2>&1 || true + kind export logs /tmp/kind + EOF - name: Upload logs - if: failure() + if: failure() || cancelled() uses: actions/upload-artifact@v4 with: name: kind-logs diff --git a/.github/scripts/deploy-greptimedb.sh b/.github/scripts/deploy-greptimedb.sh index 92ca440dbd9..9747edc7796 100755 --- a/.github/scripts/deploy-greptimedb.sh +++ b/.github/scripts/deploy-greptimedb.sh @@ -50,7 +50,7 @@ function deploy_etcd_cluster() { -n "$namespace" # Wait for etcd cluster to be ready. - kubectl rollout status statefulset/etcd -n "$namespace" + kubectl rollout status statefulset/etcd -n "$namespace" --timeout=5m } # Deploy greptimedb-operator. @@ -81,16 +81,8 @@ function deploy_greptimedb_cluster() { -n "$install_namespace" # Wait for greptimedb cluster to be ready. - while true; do - PHASE=$(kubectl -n "$install_namespace" get gtc "$cluster_name" -o jsonpath='{.status.clusterPhase}') - if [ "$PHASE" == "Running" ]; then - echo "Cluster is ready" - break - else - echo "Cluster is not ready yet: Current phase: $PHASE" - sleep 5 # wait for 5 seconds before check again. - fi - done + kubectl -n "$install_namespace" wait "gtc/$cluster_name" \ + --for=jsonpath='{.status.clusterPhase}'=Running --timeout=5m # Expose greptimedb cluster to local access and check if port-forward is available. kubectl -n "$install_namespace" port-forward svc/"$cluster_name"-frontend \ @@ -141,16 +133,8 @@ function deploy_greptimedb_cluster_with_s3_storage() { --set objectStorage.credentials.secretAccessKey="$AWS_SECRET_ACCESS_KEY" # Wait for greptimedb cluster to be ready. - while true; do - PHASE=$(kubectl -n "$install_namespace" get gtc "$cluster_name" -o jsonpath='{.status.clusterPhase}') - if [ "$PHASE" == "Running" ]; then - echo "Cluster is ready" - break - else - echo "Cluster is not ready yet: Current phase: $PHASE" - sleep 5 # wait for 5 seconds before check again. - fi - done + kubectl -n "$install_namespace" wait "gtc/$cluster_name" \ + --for=jsonpath='{.status.clusterPhase}'=Running --timeout=5m # Expose greptimedb cluster to local access. kubectl -n "$install_namespace" port-forward svc/"$cluster_name"-frontend \ @@ -168,8 +152,8 @@ function deploy_standalone_greptimedb() { --set image.tag="$GREPTIMEDB_IMAGE_TAG" \ -n "$DEFAULT_INSTALL_NAMESPACE" - # Wait for etcd cluster to be ready. - kubectl rollout status statefulset/greptimedb-standalone -n "$DEFAULT_INSTALL_NAMESPACE" + # Wait for standalone GreptimeDB to be ready. + kubectl rollout status statefulset/greptimedb-standalone -n "$DEFAULT_INSTALL_NAMESPACE" --timeout=5m # Expose greptimedb to local access. kubectl -n "$DEFAULT_INSTALL_NAMESPACE" port-forward svc/greptimedb-standalone \ diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 2b6862bb827..fe4e89cb59d 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -14,6 +14,7 @@ jobs: name: Run sqlness test if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} runs-on: ubuntu-latest + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@v4 @@ -33,7 +34,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Upload sqlness logs - if: failure() + if: failure() || cancelled() uses: actions/upload-artifact@v4 with: name: sqlness-logs-kind