diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index aef1b920ea..c8f8c681b2 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -578,8 +578,8 @@ jobs: - uses: actions/checkout@v4 - if: matrix.mode.kafka name: Setup kafka server - working-directory: tests-integration/fixtures/kafka - run: docker compose -f docker-compose-standalone.yml up -d --wait + working-directory: tests-integration/fixtures + run: docker compose up -d --wait kafka - name: Download pre-built binaries uses: actions/download-artifact@v4 with: @@ -655,31 +655,23 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 with: components: llvm-tools-preview - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - # Shares cross multiple jobs - shared-key: "coverage-test" - - name: Docker Cache - uses: ScribeMD/docker-cache@0.3.7 - with: - key: docker-${{ runner.os }}-coverage + # - name: Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # # Shares cross multiple jobs + # shared-key: "coverage-test" + # Disabled temporarily to see performance + # - name: Docker Cache + # uses: ScribeMD/docker-cache@0.5.0 + # with: + # key: docker-${{ runner.os }}-coverage - name: Install latest nextest release uses: taiki-e/install-action@nextest - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - name: Setup etcd server - working-directory: tests-integration/fixtures/etcd - run: docker compose -f docker-compose-standalone.yml up -d --wait - - name: Setup kafka server - working-directory: tests-integration/fixtures/kafka - run: docker compose -f docker-compose-standalone.yml up -d --wait - - name: Setup minio - working-directory: tests-integration/fixtures/minio - run: docker compose -f docker-compose-standalone.yml up -d --wait - - name: Setup postgres server - working-directory: tests-integration/fixtures/postgres - run: docker compose -f docker-compose-standalone.yml up -d --wait + - name: Setup external services + working-directory: tests-integration/fixtures + run: docker compose up -d --wait - name: Run nextest cases run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend env: diff --git a/tests-integration/README.md b/tests-integration/README.md index a1622174a1..9934072538 100644 --- a/tests-integration/README.md +++ b/tests-integration/README.md @@ -47,13 +47,10 @@ To run the integration test, please copy `.env.example` to `.env` in the project GT_KAFKA_ENDPOINTS = localhost:9092 ``` -### Setup kafka standalone +### Setup kafka standalone ``` -cd tests-integration/fixtures/kafka +cd tests-integration/fixtures -docker compose -f docker-compose-standalone.yml up +docker compose -f docker-compose-standalone.yml up kafka ``` - - - diff --git a/tests-integration/fixtures/docker-compose.yml b/tests-integration/fixtures/docker-compose.yml new file mode 100644 index 0000000000..7b47c2ed97 --- /dev/null +++ b/tests-integration/fixtures/docker-compose.yml @@ -0,0 +1,72 @@ +services: + + zookeeper: + image: docker.io/bitnami/zookeeper:3.7 + ports: + - '2181:2181' + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: docker.io/bitnami/kafka:3.6.0 + container_name: kafka + ports: + - 9092:9092 + - 9093:9093 + environment: + # KRaft settings + KAFKA_CFG_NODE_ID: "1" + KAFKA_CFG_PROCESS_ROLES: broker,controller + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:2181 + # Listeners + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SECURE://localhost:9093 + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SECURE:SASL_PLAINTEXT + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:2181,SECURE://:9093 + ALLOW_PLAINTEXT_LISTENER: "yes" + KAFKA_BROKER_ID: "1" + KAFKA_CLIENT_USERS: "user_kafka" + KAFKA_CLIENT_PASSWORDS: "secret" + depends_on: + zookeeper: + condition: service_started + + etcd: + image: docker.io/bitnami/etcd:3.5 + ports: + - "2379:2379" + - "2380:2380" + environment: + ALLOW_NONE_AUTHENTICATION: "yes" + ETCD_NAME: etcd + ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 + ETCD_ADVERTISE_CLIENT_URLS: http://etcd:2379 + ETCD_MAX_REQUEST_BYTES: 10485760 + + minio: + image: docker.io/bitnami/minio:2024 + ports: + - '9000:9000' + - '9001:9001' + environment: + - MINIO_ROOT_USER=superpower_ci_user + - MINIO_ROOT_PASSWORD=superpower_password + - MINIO_DEFAULT_BUCKETS=greptime + - BITNAMI_DEBUG=true + volumes: + - 'minio_data:/bitnami/minio/data' + + postgres: + image: docker.io/postgres:14-alpine + ports: + - 5432:5432 + volumes: + - ~/apps/postgres:/var/lib/postgresql/data + environment: + - POSTGRES_USER=greptimedb + - POSTGRES_DB=postgres + - POSTGRES_PASSWORD=admin + +volumes: + minio_data: + driver: local diff --git a/tests-integration/fixtures/etcd/docker-compose-standalone.yml b/tests-integration/fixtures/etcd/docker-compose-standalone.yml deleted file mode 100644 index 3c6715030a..0000000000 --- a/tests-integration/fixtures/etcd/docker-compose-standalone.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: '3.8' -services: - etcd: - image: ghcr.io/zcube/bitnami-compat/etcd:3.5 - ports: - - "2379:2379" - - "2380:2380" - environment: - ALLOW_NONE_AUTHENTICATION: "yes" - ETCD_NAME: etcd - ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 - ETCD_ADVERTISE_CLIENT_URLS: http://etcd:2379 - ETCD_MAX_REQUEST_BYTES: 10485760 diff --git a/tests-integration/fixtures/kafka/README.md b/tests-integration/fixtures/kafka/README.md deleted file mode 100644 index 9d49a22893..0000000000 --- a/tests-integration/fixtures/kafka/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Starts a standalone kafka -```bash -docker compose -f docker-compose-standalone.yml up kafka -d -``` - -## Lists running services -```bash -docker compose -f docker-compose-standalone.yml ps -``` - -## Stops the standalone kafka -```bash -docker compose -f docker-compose-standalone.yml stop kafka -``` - -## Stops and removes the standalone kafka -```bash -docker compose -f docker-compose-standalone.yml down kafka -``` \ No newline at end of file diff --git a/tests-integration/fixtures/kafka/docker-compose-standalone.yml b/tests-integration/fixtures/kafka/docker-compose-standalone.yml deleted file mode 100644 index 4208fe3f67..0000000000 --- a/tests-integration/fixtures/kafka/docker-compose-standalone.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: '3.8' -services: - zookeeper: - image: bitnami/zookeeper:3.7 - ports: - - '2181:2181' - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - kafka: - image: bitnami/kafka:3.6.0 - container_name: kafka - ports: - - 9092:9092 - - 9093:9093 - environment: - # KRaft settings - KAFKA_CFG_NODE_ID: "1" - KAFKA_CFG_PROCESS_ROLES: broker,controller - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:2181 - # Listeners - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SECURE://localhost:9093 - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SECURE:SASL_PLAINTEXT - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:2181,SECURE://:9093 - ALLOW_PLAINTEXT_LISTENER: "yes" - KAFKA_BROKER_ID: "1" - KAFKA_CLIENT_USERS: "user_kafka" - KAFKA_CLIENT_PASSWORDS: "secret" diff --git a/tests-integration/fixtures/minio/docker-compose-standalone.yml b/tests-integration/fixtures/minio/docker-compose-standalone.yml deleted file mode 100644 index 139cb916a9..0000000000 --- a/tests-integration/fixtures/minio/docker-compose-standalone.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: '3.8' -services: - minio: - image: bitnami/minio:2024 - ports: - - '9000:9000' - - '9001:9001' - environment: - - MINIO_ROOT_USER=superpower_ci_user - - MINIO_ROOT_PASSWORD=superpower_password - - MINIO_DEFAULT_BUCKETS=greptime - - BITNAMI_DEBUG=true - volumes: - - 'minio_data:/bitnami/minio/data' - -volumes: - minio_data: - driver: local diff --git a/tests-integration/fixtures/postgres/docker-compose-standalone.yml b/tests-integration/fixtures/postgres/docker-compose-standalone.yml deleted file mode 100644 index 4f1ca8e007..0000000000 --- a/tests-integration/fixtures/postgres/docker-compose-standalone.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: '3.9' -services: - postgres: - image: postgres:14-alpine - ports: - - 5432:5432 - volumes: - - ~/apps/postgres:/var/lib/postgresql/data - environment: - - POSTGRES_USER=greptimedb - - POSTGRES_DB=postgres - - POSTGRES_PASSWORD=admin