diff --git a/.github/actions/setup-etcd-cluster/action.yml b/.github/actions/setup-etcd-cluster/action.yml index e6abb96bc2..886880dd20 100644 --- a/.github/actions/setup-etcd-cluster/action.yml +++ b/.github/actions/setup-etcd-cluster/action.yml @@ -12,7 +12,7 @@ runs: steps: - name: Install Etcd cluster shell: bash - run: | + run: | helm upgrade \ --install etcd oci://registry-1.docker.io/bitnamicharts/etcd \ --set replicaCount=${{ inputs.etcd-replicas }} \ @@ -24,4 +24,9 @@ runs: --set auth.rbac.token.enabled=false \ --set persistence.size=2Gi \ --create-namespace \ + --set global.security.allowInsecureImages=true \ + --set image.registry=public.ecr.aws/i8k6a5e1 \ + --set image.repository=bitnami/etcd \ + --set image.tag=3.6.1-debian-12-r3 \ + --version 12.0.8 \ -n ${{ inputs.namespace }} diff --git a/.github/actions/setup-kafka-cluster/action.yml b/.github/actions/setup-kafka-cluster/action.yml index 22b4389957..3eeb13f4d7 100644 --- a/.github/actions/setup-kafka-cluster/action.yml +++ b/.github/actions/setup-kafka-cluster/action.yml @@ -12,7 +12,7 @@ runs: steps: - name: Install Kafka cluster shell: bash - run: | + run: | helm upgrade \ --install kafka oci://registry-1.docker.io/bitnamicharts/kafka \ --set controller.replicaCount=${{ inputs.controller-replicas }} \ @@ -23,4 +23,8 @@ runs: --set listeners.controller.protocol=PLAINTEXT \ --set listeners.client.protocol=PLAINTEXT \ --create-namespace \ + --set image.registry=public.ecr.aws/i8k6a5e1 \ + --set image.repository=bitnami/kafka \ + --set image.tag=3.9.0-debian-12-r1 \ + --version 31.0.0 \ -n ${{ inputs.namespace }} diff --git a/.github/actions/setup-postgres-cluster/action.yml b/.github/actions/setup-postgres-cluster/action.yml index 5f93a6a63e..62aab57852 100644 --- a/.github/actions/setup-postgres-cluster/action.yml +++ b/.github/actions/setup-postgres-cluster/action.yml @@ -6,9 +6,7 @@ inputs: description: "Number of PostgreSQL replicas" namespace: default: "postgres-namespace" - postgres-version: - default: "14.2" - description: "PostgreSQL version" + description: "The PostgreSQL namespace" storage-size: default: "1Gi" description: "Storage size for PostgreSQL" @@ -22,7 +20,11 @@ runs: helm upgrade \ --install postgresql oci://registry-1.docker.io/bitnamicharts/postgresql \ --set replicaCount=${{ inputs.postgres-replicas }} \ - --set image.tag=${{ inputs.postgres-version }} \ + --set global.security.allowInsecureImages=true \ + --set image.registry=public.ecr.aws/i8k6a5e1 \ + --set image.repository=bitnami/postgresql \ + --set image.tag=17.5.0-debian-12-r3 \ + --version 16.7.4 \ --set persistence.size=${{ inputs.storage-size }} \ --set postgresql.username=greptimedb \ --set postgresql.password=admin \ diff --git a/.github/scripts/pull-test-deps-images.sh b/.github/scripts/pull-test-deps-images.sh new file mode 100755 index 0000000000..968ac44079 --- /dev/null +++ b/.github/scripts/pull-test-deps-images.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script is used to pull the test dependency images that are stored in public ECR one by one to avoid rate limiting. + +set -e + +MAX_RETRIES=3 + +IMAGES=( + "public.ecr.aws/i8k6a5e1/bitnami/zookeeper:3.7" + "public.ecr.aws/i8k6a5e1/bitnami/kafka:3.9.0-debian-12-r1" + "public.ecr.aws/i8k6a5e1/bitnami/etcd:3.6.1-debian-12-r3" + "public.ecr.aws/i8k6a5e1/bitnami/minio:2024" + "public.ecr.aws/i8k6a5e1/bitnami/mysql:5.7" +) + +for image in "${IMAGES[@]}"; do + for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do + if docker pull "$image"; then + # Successfully pulled the image. + break + else + # Use some simple exponential backoff to avoid rate limiting. + if [ $attempt -lt $MAX_RETRIES ]; then + sleep_seconds=$((attempt * 5)) + echo "Attempt $attempt failed for $image, waiting $sleep_seconds seconds" + sleep $sleep_seconds # 5s, 10s delays + else + echo "Failed to pull $image after $MAX_RETRIES attempts" + exit 1 + fi + fi + done +done diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index dfd2ccaab3..f09f0ceb03 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -744,6 +744,10 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install latest nextest release uses: taiki-e/install-action@nextest + + - name: Pull test dependencies images + run: ./.github/scripts/pull-test-deps-images.sh + - name: Setup external services working-directory: tests-integration/fixtures run: docker compose up -d --wait diff --git a/tests-integration/fixtures/docker-compose.yml b/tests-integration/fixtures/docker-compose.yml index 164331f692..366bf69482 100644 --- a/tests-integration/fixtures/docker-compose.yml +++ b/tests-integration/fixtures/docker-compose.yml @@ -1,14 +1,14 @@ services: zookeeper: - image: docker.io/bitnami/zookeeper:3.7 + image: public.ecr.aws/i8k6a5e1/bitnami/zookeeper:3.7 ports: - '2181:2181' environment: - ALLOW_ANONYMOUS_LOGIN=yes kafka: - image: docker.io/bitnami/kafka:3.9.0 + image: public.ecr.aws/i8k6a5e1/bitnami/kafka:3.9.0-debian-12-r1 container_name: kafka ports: - 9092:9092 @@ -32,7 +32,7 @@ services: condition: service_started etcd: - image: docker.io/bitnami/etcd:3.5 + image: public.ecr.aws/i8k6a5e1/bitnami/etcd:3.6.1-debian-12-r3 ports: - "2379:2379" - "2380:2380" @@ -44,7 +44,7 @@ services: ETCD_MAX_REQUEST_BYTES: 10485760 etcd-tls: - image: docker.io/bitnami/etcd:3.5 + image: public.ecr.aws/i8k6a5e1/bitnami/etcd:3.6.1-debian-12-r3 ports: - "2378:2378" - "2381:2381" @@ -71,7 +71,7 @@ services: - ./etcd-tls-certs:/certs:ro minio: - image: docker.io/bitnami/minio:2024 + image: public.ecr.aws/i8k6a5e1/bitnami/minio:2024 ports: - '9000:9000' - '9001:9001' @@ -108,7 +108,7 @@ services: mysql: - image: bitnami/mysql:5.7 + image: public.ecr.aws/i8k6a5e1/bitnami/mysql:5.7 ports: - 3306:3306 volumes: diff --git a/tests/conf/kafka-cluster.yml b/tests/conf/kafka-cluster.yml index b928f9e1ec..c205bafe3d 100644 --- a/tests/conf/kafka-cluster.yml +++ b/tests/conf/kafka-cluster.yml @@ -2,7 +2,7 @@ version: '3.8' services: kafka: - image: bitnami/kafka:3.6.0 + image: public.ecr.aws/i8k6a5e1/bitnami/kafka:3.9.0-debian-12-r1 container_name: kafka ports: - 9092:9092 diff --git a/tests/runner/src/util.rs b/tests/runner/src/util.rs index 7f7de8500a..e4ba482de1 100644 --- a/tests/runner/src/util.rs +++ b/tests/runner/src/util.rs @@ -363,9 +363,9 @@ pub fn setup_mysql(mysql_port: u16, mysql_version: Option<&str>) { } let mysql_image = if let Some(mysql_version) = mysql_version { - format!("bitnami/mysql:{mysql_version}") + format!("public.ecr.aws/i8k6a5e1/bitnami/mysql:{mysql_version}") } else { - "bitnami/mysql:5.7".to_string() + "public.ecr.aws/i8k6a5e1/bitnami/mysql:5.7".to_string() }; let mysql_password = "admin"; let mysql_user = "greptimedb";