diff --git a/.github/actions/setup-kafka-cluster/action.yml b/.github/actions/setup-kafka-cluster/action.yml index fca420da70..ef319866b3 100644 --- a/.github/actions/setup-kafka-cluster/action.yml +++ b/.github/actions/setup-kafka-cluster/action.yml @@ -6,6 +6,9 @@ inputs: description: "Kafka controller replicas" namespace: default: "kafka-cluster" + provider: + default: "kubernetes" + description: "Deployment provider: kubernetes or docker" runs: using: composite @@ -13,6 +16,13 @@ runs: - name: Install Kafka cluster shell: bash run: | + if [ "${{ inputs.provider }}" = "docker" ]; then + REPO_ROOT=$(git rev-parse --show-toplevel) + "${REPO_ROOT}/.github/scripts/pull-test-deps-images.sh" + docker compose -f "${REPO_ROOT}/tests-integration/fixtures/docker-compose.yml" up -d --wait kafka + exit 0 + fi + helm upgrade \ --install kafka oci://registry-1.docker.io/bitnamicharts/kafka \ --set controller.replicaCount=${{ inputs.controller-replicas }} \ diff --git a/.github/actions/setup-mysql-cluster/action.yml b/.github/actions/setup-mysql-cluster/action.yml new file mode 100644 index 0000000000..8fd9eba686 --- /dev/null +++ b/.github/actions/setup-mysql-cluster/action.yml @@ -0,0 +1,21 @@ +name: Setup MySQL +description: Deploy MySQL on Docker or Kubernetes +inputs: + provider: + default: "docker" + description: "Deployment provider: docker" + +runs: + using: composite + steps: + - name: Install MySQL + shell: bash + run: | + if [ "${{ inputs.provider }}" != "docker" ]; then + echo "Unsupported provider: ${{ inputs.provider }}" + exit 1 + fi + + REPO_ROOT=$(git rev-parse --show-toplevel) + "${REPO_ROOT}/.github/scripts/pull-test-deps-images.sh" + docker compose -f "${REPO_ROOT}/tests-integration/fixtures/docker-compose.yml" up -d --wait mysql diff --git a/.github/actions/setup-postgres-cluster/action.yml b/.github/actions/setup-postgres-cluster/action.yml index 4f40bf51d3..f3bb525f4d 100644 --- a/.github/actions/setup-postgres-cluster/action.yml +++ b/.github/actions/setup-postgres-cluster/action.yml @@ -10,6 +10,9 @@ inputs: storage-size: default: "1Gi" description: "Storage size for PostgreSQL" + provider: + default: "kubernetes" + description: "Deployment provider: kubernetes or docker" runs: using: composite @@ -17,6 +20,12 @@ runs: - name: Install PostgreSQL shell: bash run: | + if [ "${{ inputs.provider }}" = "docker" ]; then + REPO_ROOT=$(git rev-parse --show-toplevel) + docker compose -f "${REPO_ROOT}/tests-integration/fixtures/docker-compose.yml" up -d --wait postgres + exit 0 + fi + helm upgrade \ --install postgresql oci://registry-1.docker.io/bitnamicharts/postgresql \ --set replicaCount=${{ inputs.postgres-replicas }} \ diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 69f0a8ea34..4b0231b074 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -669,81 +669,49 @@ jobs: - name: "Basic" opts: "" kafka: false + postgres: false + mysql: false - name: "Remote WAL" opts: "-w kafka -k 127.0.0.1:9092" kafka: true + postgres: false + mysql: false - name: "PostgreSQL KvBackend" opts: "--setup-pg postgresql://greptimedb:admin@127.0.0.1:5432/postgres" kafka: false + postgres: true + mysql: false - name: "MySQL KvBackend" opts: "--setup-mysql mysql://greptimedb:admin@127.0.0.1:3306/mysql" kafka: false + postgres: false + mysql: true - name: "Flat format" opts: "--enable-flat-format" kafka: false + postgres: false + mysql: false timeout-minutes: 60 - services: - kafka: - image: greptime/kafka:3.9.0-debian-12-r1 - ports: - - 9092:9092 - env: - KAFKA_KRAFT_CLUSTER_ID: Kmp-xkTnSf-WWXhWmiorDg - KAFKA_ENABLE_KRAFT: "yes" - KAFKA_CFG_NODE_ID: "1" - KAFKA_CFG_PROCESS_ROLES: broker,controller - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:2181 - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092 - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:2181 - ALLOW_PLAINTEXT_LISTENER: "yes" - KAFKA_BROKER_ID: "1" - postgres: - image: docker.io/postgres:14-alpine - env: - POSTGRES_USER: greptimedb - POSTGRES_DB: postgres - POSTGRES_PASSWORD: admin - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U greptimedb -d postgres" - --health-interval 5s - --health-timeout 5s - --health-retries 12 - mysql: - image: greptime/mysql:5.7 - env: - MYSQL_DATABASE: mysql - MYSQL_USER: greptimedb - MYSQL_PASSWORD: admin - MYSQL_ROOT_PASSWORD: admin - ports: - - 3306:3306 - options: >- - --health-cmd "mysqladmin ping -h 127.0.0.1 -u root -padmin --silent" - --health-interval 5s - --health-timeout 5s - --health-retries 12 steps: - uses: actions/checkout@v4 with: persist-credentials: false - if: matrix.mode.kafka - name: Wait for kafka server - shell: bash - run: | - # Allow up to 60 seconds for the service container to accept connections. - for _ in {1..30}; do - if (echo > /dev/tcp/127.0.0.1/9092) >/dev/null 2>&1; then - exit 0 - fi - sleep 2 - done - echo "Kafka is not ready" - exit 1 + name: Setup Kafka cluster + uses: ./.github/actions/setup-kafka-cluster + with: + provider: docker + + - if: matrix.mode.postgres + name: Setup PostgreSQL + uses: ./.github/actions/setup-postgres-cluster + with: + provider: docker + + - if: matrix.mode.mysql + name: Setup MySQL + uses: ./.github/actions/setup-mysql-cluster - name: Download pre-built binaries uses: actions/download-artifact@v4