ci: disable docker/rust cache temporarily and merge docker compose files (#5293)

* ci: disable docker cache temporarily and merge docker compose files

* ci: fix compose file name and options

* ci: try to disable rust cache

.github/workflows/develop.yml

@@ -578,8 +578,8 @@ jobs:
       - uses: actions/checkout@v4
       - if: matrix.mode.kafka
         name: Setup kafka server
-        working-directory: tests-integration/fixtures/kafka
+        working-directory: tests-integration/fixtures
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
+        run: docker compose up -d --wait kafka
       - name: Download pre-built binaries
         uses: actions/download-artifact@v4
         with:
@@ -655,31 +655,23 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           components: llvm-tools-preview
-      - name: Rust Cache
+      # - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
+      #   uses: Swatinem/rust-cache@v2
-        with:
+      #   with:
-          # Shares cross multiple jobs
+      #     # Shares cross multiple jobs
-          shared-key: "coverage-test"
+      #     shared-key: "coverage-test"
-      - name: Docker Cache
+      # Disabled temporarily to see performance
-        uses: ScribeMD/docker-cache@0.3.7
+      # - name: Docker Cache
-        with:
+      #   uses: ScribeMD/docker-cache@0.5.0
-          key: docker-${{ runner.os }}-coverage
+      #   with:
+      #     key: docker-${{ runner.os }}-coverage
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
-      - name: Setup etcd server
+      - name: Setup external services
-        working-directory: tests-integration/fixtures/etcd
+        working-directory: tests-integration/fixtures
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
+        run: docker compose up -d --wait
-      - name: Setup kafka server
-        working-directory: tests-integration/fixtures/kafka
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup minio
-        working-directory: tests-integration/fixtures/minio
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup postgres server
-        working-directory: tests-integration/fixtures/postgres
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend
         env:

tests-integration/README.md

@@ -50,10 +50,7 @@ GT_KAFKA_ENDPOINTS = localhost:9092
 ### Setup kafka standalone
 
 ```
-cd tests-integration/fixtures/kafka 
+cd tests-integration/fixtures
 
-docker compose -f docker-compose-standalone.yml up
+docker compose -f docker-compose-standalone.yml up kafka
 ```
-
-
-

tests-integration/fixtures/docker-compose.yml

@@ -0,0 +1,72 @@
+services:
+
+  zookeeper:
+    image: docker.io/bitnami/zookeeper:3.7
+    ports:
+      - '2181:2181'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+
+  kafka:
+    image: docker.io/bitnami/kafka:3.6.0
+    container_name: kafka
+    ports:
+      - 9092:9092
+      - 9093:9093
+    environment:
+      # KRaft settings
+      KAFKA_CFG_NODE_ID: "1"
+      KAFKA_CFG_PROCESS_ROLES: broker,controller
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:2181
+      # Listeners
+      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SECURE://localhost:9093
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SECURE:SASL_PLAINTEXT
+      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:2181,SECURE://:9093
+      ALLOW_PLAINTEXT_LISTENER: "yes"
+      KAFKA_BROKER_ID: "1"
+      KAFKA_CLIENT_USERS: "user_kafka"
+      KAFKA_CLIENT_PASSWORDS: "secret"
+    depends_on:
+      zookeeper:
+        condition: service_started
+
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    ports:
+      - "2379:2379"
+      - "2380:2380"
+    environment:
+      ALLOW_NONE_AUTHENTICATION: "yes"
+      ETCD_NAME: etcd
+      ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
+      ETCD_ADVERTISE_CLIENT_URLS: http://etcd:2379
+      ETCD_MAX_REQUEST_BYTES: 10485760
+
+  minio:
+    image: docker.io/bitnami/minio:2024
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      - MINIO_ROOT_USER=superpower_ci_user
+      - MINIO_ROOT_PASSWORD=superpower_password
+      - MINIO_DEFAULT_BUCKETS=greptime
+      - BITNAMI_DEBUG=true
+    volumes:
+      - 'minio_data:/bitnami/minio/data'
+
+  postgres:
+    image: docker.io/postgres:14-alpine
+    ports:
+      - 5432:5432
+    volumes:
+      - ~/apps/postgres:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=greptimedb
+      - POSTGRES_DB=postgres
+      - POSTGRES_PASSWORD=admin
+
+volumes:
+  minio_data:
+    driver: local

tests-integration/fixtures/etcd/docker-compose-standalone.yml

@@ -1,13 +0,0 @@
-version: '3.8'
-services:
-  etcd:
-    image: ghcr.io/zcube/bitnami-compat/etcd:3.5
-    ports:
-      - "2379:2379"
-      - "2380:2380"
-    environment:
-      ALLOW_NONE_AUTHENTICATION: "yes"
-      ETCD_NAME: etcd
-      ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
-      ETCD_ADVERTISE_CLIENT_URLS: http://etcd:2379
-      ETCD_MAX_REQUEST_BYTES: 10485760

tests-integration/fixtures/kafka/README.md

@@ -1,19 +0,0 @@
-## Starts a standalone kafka 
-```bash
-docker compose -f docker-compose-standalone.yml up kafka -d
-```
-
-## Lists running services
-```bash
-docker compose -f docker-compose-standalone.yml ps
-```
-
-## Stops the standalone kafka 
-```bash
-docker compose -f docker-compose-standalone.yml stop kafka 
-```
-
-## Stops and removes the standalone kafka 
-```bash
-docker compose -f docker-compose-standalone.yml down kafka 
-```

tests-integration/fixtures/kafka/docker-compose-standalone.yml

@@ -1,28 +0,0 @@
-version: '3.8'
-services:
-  zookeeper:
-    image: bitnami/zookeeper:3.7
-    ports:
-      - '2181:2181'
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
-  kafka:
-    image: bitnami/kafka:3.6.0
-    container_name: kafka
-    ports:
-      - 9092:9092
-      - 9093:9093
-    environment:
-      # KRaft settings
-      KAFKA_CFG_NODE_ID: "1"
-      KAFKA_CFG_PROCESS_ROLES: broker,controller
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:2181
-      # Listeners
-      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SECURE://localhost:9093
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SECURE:SASL_PLAINTEXT
-      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:2181,SECURE://:9093
-      ALLOW_PLAINTEXT_LISTENER: "yes"
-      KAFKA_BROKER_ID: "1"
-      KAFKA_CLIENT_USERS: "user_kafka"
-      KAFKA_CLIENT_PASSWORDS: "secret"

tests-integration/fixtures/minio/docker-compose-standalone.yml

@@ -1,18 +0,0 @@
-version: '3.8'
-services:
-  minio:
-    image: bitnami/minio:2024
-    ports:
-      - '9000:9000'
-      - '9001:9001'
-    environment:
-      - MINIO_ROOT_USER=superpower_ci_user
-      - MINIO_ROOT_PASSWORD=superpower_password
-      - MINIO_DEFAULT_BUCKETS=greptime
-      - BITNAMI_DEBUG=true
-    volumes:
-      - 'minio_data:/bitnami/minio/data'
-
-volumes:
-  minio_data:
-    driver: local

tests-integration/fixtures/postgres/docker-compose-standalone.yml

@@ -1,12 +0,0 @@
-version: '3.9'
-services:
-  postgres:
-    image: postgres:14-alpine
-    ports:
-      - 5432:5432
-    volumes:
-      - ~/apps/postgres:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=greptimedb
-      - POSTGRES_DB=postgres
-      - POSTGRES_PASSWORD=admin