diff --git a/.github/actions/setup-etcd-cluster/action.yml b/.github/actions/setup-etcd-cluster/action.yml index e6abb96bc2..b69c1c6e2c 100644 --- a/.github/actions/setup-etcd-cluster/action.yml +++ b/.github/actions/setup-etcd-cluster/action.yml @@ -24,4 +24,9 @@ runs: --set auth.rbac.token.enabled=false \ --set persistence.size=2Gi \ --create-namespace \ + --set global.security.allowInsecureImages=true \ + --set image.registry=docker.io \ + --set image.repository=greptime/etcd \ + --set image.tag=3.6.1-debian-12-r3 \ + --version 12.0.8 \ -n ${{ inputs.namespace }} diff --git a/.github/actions/setup-greptimedb-cluster/action.yml b/.github/actions/setup-greptimedb-cluster/action.yml index 7c385c43a9..59163424f4 100644 --- a/.github/actions/setup-greptimedb-cluster/action.yml +++ b/.github/actions/setup-greptimedb-cluster/action.yml @@ -51,7 +51,7 @@ runs: run: | helm upgrade \ --install my-greptimedb \ - --set meta.etcdEndpoints=${{ inputs.etcd-endpoints }} \ + --set meta.backendStorage.etcd.endpoints=${{ inputs.etcd-endpoints }} \ --set meta.enableRegionFailover=${{ inputs.enable-region-failover }} \ --set image.registry=${{ inputs.image-registry }} \ --set image.repository=${{ inputs.image-repository }} \ diff --git a/.github/actions/setup-kafka-cluster/action.yml b/.github/actions/setup-kafka-cluster/action.yml index 22b4389957..50183a4ab9 100644 --- a/.github/actions/setup-kafka-cluster/action.yml +++ b/.github/actions/setup-kafka-cluster/action.yml @@ -23,4 +23,8 @@ runs: --set listeners.controller.protocol=PLAINTEXT \ --set listeners.client.protocol=PLAINTEXT \ --create-namespace \ + --set image.registry=docker.io \ + --set image.repository=greptime/kafka \ + --set image.tag=3.9.0-debian-12-r1 \ + --version 31.0.0 \ -n ${{ inputs.namespace }} diff --git a/.github/scripts/pull-test-deps-images.sh b/.github/scripts/pull-test-deps-images.sh new file mode 100755 index 0000000000..3d0d712c51 --- /dev/null +++ b/.github/scripts/pull-test-deps-images.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script is used to pull the test dependency images that are stored in public ECR one by one to avoid rate limiting. + +set -e + +MAX_RETRIES=3 + +IMAGES=( + "greptime/zookeeper:3.7" + "greptime/kafka:3.9.0-debian-12-r1" + "greptime/etcd:3.6.1-debian-12-r3" + "greptime/minio:2024" + "greptime/mysql:5.7" +) + +for image in "${IMAGES[@]}"; do + for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do + if docker pull "$image"; then + # Successfully pulled the image. + break + else + # Use some simple exponential backoff to avoid rate limiting. + if [ $attempt -lt $MAX_RETRIES ]; then + sleep_seconds=$((attempt * 5)) + echo "Attempt $attempt failed for $image, waiting $sleep_seconds seconds" + sleep $sleep_seconds # 5s, 10s delays + else + echo "Failed to pull $image after $MAX_RETRIES attempts" + exit 1 + fi + fi + done +done diff --git a/.github/workflows/apidoc.yml b/.github/workflows/apidoc.yml index 6bbca5d9a2..eaa7989e8f 100644 --- a/.github/workflows/apidoc.yml +++ b/.github/workflows/apidoc.yml @@ -14,7 +14,7 @@ name: Build API docs jobs: apidoc: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 6697e7331b..ff804ba39e 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -16,11 +16,11 @@ on: description: The runner uses to build linux-amd64 artifacts default: ec2-c6i.4xlarge-amd64 options: - - ubuntu-20.04 - - ubuntu-20.04-8-cores - - ubuntu-20.04-16-cores - - ubuntu-20.04-32-cores - - ubuntu-20.04-64-cores + - ubuntu-22.04 + - ubuntu-22.04-8-cores + - ubuntu-22.04-16-cores + - ubuntu-22.04-32-cores + - ubuntu-22.04-64-cores - ec2-c6i.xlarge-amd64 # 4C8G - ec2-c6i.2xlarge-amd64 # 8C16G - ec2-c6i.4xlarge-amd64 # 16C32G @@ -83,7 +83,7 @@ jobs: allocate-runners: name: Allocate runners if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }} linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} @@ -218,7 +218,7 @@ jobs: build-linux-amd64-artifacts, build-linux-arm64-artifacts, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: build-result: ${{ steps.set-build-result.outputs.build-result }} steps: @@ -251,7 +251,7 @@ jobs: allocate-runners, release-images-to-dockerhub, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 continue-on-error: true steps: - uses: actions/checkout@v4 @@ -283,7 +283,7 @@ jobs: name: Stop linux-amd64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-amd64-artifacts, @@ -309,7 +309,7 @@ jobs: name: Stop linux-arm64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-arm64-artifacts, @@ -337,7 +337,7 @@ jobs: needs: [ release-images-to-dockerhub ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 permissions: issues: write diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index fb454b1857..d4d21508ce 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -21,14 +21,13 @@ concurrency: cancel-in-progress: true jobs: - check-typos-and-docs: - name: Check typos and docs - runs-on: ubuntu-20.04 + check-docs: + name: Check docs + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: persist-credentials: false - - uses: crate-ci/typos@master - name: Check the config docs run: | make config-docs && \ @@ -36,7 +35,7 @@ jobs: || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1) license-header-check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 name: Check License Header steps: - uses: actions/checkout@v4 @@ -49,7 +48,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ ubuntu-22.04 ] timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -72,7 +71,7 @@ jobs: toml: name: Toml Check - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -89,7 +88,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ ubuntu-22.04 ] timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -248,7 +247,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ ubuntu-22.04 ] timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -568,7 +567,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ ubuntu-22.04 ] mode: - name: "Basic" opts: "" @@ -587,7 +586,8 @@ jobs: - if: matrix.mode.kafka name: Setup kafka server working-directory: tests-integration/fixtures - run: docker compose up -d --wait kafka + run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait kafka + - name: Download pre-built binaries uses: actions/download-artifact@v4 with: @@ -607,7 +607,7 @@ jobs: fmt: name: Rustfmt - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -624,7 +624,7 @@ jobs: clippy: name: Clippy - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -685,7 +685,8 @@ jobs: uses: taiki-e/install-action@nextest - name: Setup external services working-directory: tests-integration/fixtures - run: docker compose up -d --wait + run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait + - name: Run nextest cases run: cargo nextest run --workspace -F dashboard -F pg_kvbackend env: @@ -710,7 +711,7 @@ jobs: coverage: if: github.event_name == 'merge_group' - runs-on: ubuntu-20.04-8-cores + runs-on: ubuntu-22.04-8-cores timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -737,7 +738,8 @@ jobs: uses: taiki-e/install-action@cargo-llvm-cov - name: Setup external services working-directory: tests-integration/fixtures - run: docker compose up -d --wait + run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait + - name: Run nextest cases run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend env: @@ -770,7 +772,7 @@ jobs: # compat: # name: Compatibility Test # needs: build - # runs-on: ubuntu-20.04 + # runs-on: ubuntu-22.04 # timeout-minutes: 60 # steps: # - uses: actions/checkout@v4 diff --git a/.github/workflows/docbot.yml b/.github/workflows/docbot.yml index 887c8c5942..11455c767a 100644 --- a/.github/workflows/docbot.yml +++ b/.github/workflows/docbot.yml @@ -5,7 +5,7 @@ on: jobs: docbot: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 permissions: pull-requests: write contents: read diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 61f78a84fd..7eece20d4b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,7 +31,7 @@ name: CI jobs: typos: name: Spell Check with Typos - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -39,7 +39,7 @@ jobs: - uses: crate-ci/typos@master license-header-check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 name: Check License Header steps: - uses: actions/checkout@v4 @@ -49,29 +49,29 @@ jobs: check: name: Check - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: 'echo "No action required"' fmt: name: Rustfmt - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: 'echo "No action required"' clippy: name: Clippy - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: 'echo "No action required"' coverage: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: 'echo "No action required"' test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - run: 'echo "No action required"' @@ -80,7 +80,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-20.04 ] + os: [ ubuntu-22.04 ] mode: - name: "Basic" - name: "Remote WAL" diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index b0b7f3fbe3..9102fa81ff 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -14,11 +14,11 @@ on: description: The runner uses to build linux-amd64 artifacts default: ec2-c6i.4xlarge-amd64 options: - - ubuntu-20.04 - - ubuntu-20.04-8-cores - - ubuntu-20.04-16-cores - - ubuntu-20.04-32-cores - - ubuntu-20.04-64-cores + - ubuntu-22.04 + - ubuntu-22.04-8-cores + - ubuntu-22.04-16-cores + - ubuntu-22.04-32-cores + - ubuntu-22.04-64-cores - ec2-c6i.xlarge-amd64 # 4C8G - ec2-c6i.2xlarge-amd64 # 8C16G - ec2-c6i.4xlarge-amd64 # 16C32G @@ -70,7 +70,7 @@ jobs: allocate-runners: name: Allocate runners if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }} linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} @@ -182,7 +182,7 @@ jobs: build-linux-amd64-artifacts, build-linux-arm64-artifacts, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: nightly-build-result: ${{ steps.set-nightly-build-result.outputs.nightly-build-result }} steps: @@ -214,7 +214,7 @@ jobs: allocate-runners, release-images-to-dockerhub, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # When we push to ACR, it's easy to fail due to some unknown network issues. # However, we don't want to fail the whole workflow because of this. # The ACR have daily sync with DockerHub, so don't worry about the image not being updated. @@ -249,7 +249,7 @@ jobs: name: Stop linux-amd64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-amd64-artifacts, @@ -275,7 +275,7 @@ jobs: name: Stop linux-arm64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-arm64-artifacts, @@ -303,7 +303,7 @@ jobs: needs: [ release-images-to-dockerhub ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 permissions: issues: write env: diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 041e1ef673..4ba1e67c6f 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -133,7 +133,7 @@ jobs: name: Check status needs: [sqlness-test, sqlness-windows, test-on-windows] if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: check-result: ${{ steps.set-check-result.outputs.check-result }} steps: @@ -146,7 +146,7 @@ jobs: if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run. name: Send notification to Greptime team needs: [check-status] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} steps: diff --git a/.github/workflows/release-dev-builder-images.yaml b/.github/workflows/release-dev-builder-images.yaml index 2e60736140..90a3d33f50 100644 --- a/.github/workflows/release-dev-builder-images.yaml +++ b/.github/workflows/release-dev-builder-images.yaml @@ -29,7 +29,7 @@ jobs: release-dev-builder-images: name: Release dev builder images if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job. - runs-on: ubuntu-20.04-16-cores + runs-on: ubuntu-22.04-16-cores outputs: version: ${{ steps.set-version.outputs.version }} steps: @@ -63,7 +63,7 @@ jobs: release-dev-builder-images-ecr: name: Release dev builder images to AWS ECR - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ release-dev-builder-images ] @@ -148,7 +148,7 @@ jobs: release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container. name: Release dev builder images to CN region - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ release-dev-builder-images ] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81e034d257..f89d49b0dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,11 +18,11 @@ on: description: The runner uses to build linux-amd64 artifacts default: ec2-c6i.4xlarge-amd64 options: - - ubuntu-20.04 - - ubuntu-20.04-8-cores - - ubuntu-20.04-16-cores - - ubuntu-20.04-32-cores - - ubuntu-20.04-64-cores + - ubuntu-22.04 + - ubuntu-22.04-8-cores + - ubuntu-22.04-16-cores + - ubuntu-22.04-32-cores + - ubuntu-22.04-64-cores - ec2-c6i.xlarge-amd64 # 4C8G - ec2-c6i.2xlarge-amd64 # 8C16G - ec2-c6i.4xlarge-amd64 # 16C32G @@ -97,7 +97,7 @@ jobs: allocate-runners: name: Allocate runners if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }} linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} @@ -335,7 +335,7 @@ jobs: build-windows-artifacts, release-images-to-dockerhub, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # When we push to ACR, it's easy to fail due to some unknown network issues. # However, we don't want to fail the whole workflow because of this. # The ACR have daily sync with DockerHub, so don't worry about the image not being updated. @@ -377,7 +377,7 @@ jobs: build-windows-artifacts, release-images-to-dockerhub, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -396,7 +396,7 @@ jobs: name: Stop linux-amd64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-amd64-artifacts, @@ -422,7 +422,7 @@ jobs: name: Stop linux-arm64 runner # Only run this job when the runner is allocated. if: ${{ always() }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [ allocate-runners, build-linux-arm64-artifacts, @@ -448,7 +448,7 @@ jobs: name: Bump doc version if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} needs: [allocate-runners] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs permissions: issues: write # Allows the action to create issues for cyborg. @@ -475,7 +475,7 @@ jobs: build-macos-artifacts, build-windows-artifacts, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # Permission reference: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs permissions: issues: write # Allows the action to create issues for cyborg. diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml index a8fd6d05f4..9d2989789f 100644 --- a/.github/workflows/semantic-pull-request.yml +++ b/.github/workflows/semantic-pull-request.yml @@ -9,7 +9,7 @@ on: jobs: check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/tests-integration/fixtures/docker-compose.yml b/tests-integration/fixtures/docker-compose.yml index 7b47c2ed97..bc56ca3b6d 100644 --- a/tests-integration/fixtures/docker-compose.yml +++ b/tests-integration/fixtures/docker-compose.yml @@ -1,14 +1,14 @@ services: zookeeper: - image: docker.io/bitnami/zookeeper:3.7 + image: greptime/zookeeper:3.7 ports: - '2181:2181' environment: - ALLOW_ANONYMOUS_LOGIN=yes kafka: - image: docker.io/bitnami/kafka:3.6.0 + image: greptime/kafka:3.9.0-debian-12-r1 container_name: kafka ports: - 9092:9092 @@ -32,7 +32,7 @@ services: condition: service_started etcd: - image: docker.io/bitnami/etcd:3.5 + image: greptime/etcd:3.6.1-debian-12-r3 ports: - "2379:2379" - "2380:2380" @@ -44,7 +44,7 @@ services: ETCD_MAX_REQUEST_BYTES: 10485760 minio: - image: docker.io/bitnami/minio:2024 + image: greptime/minio:2024 ports: - '9000:9000' - '9001:9001'