ci: add remote WAL logical pruning fuzz (#8307)

* feat: configure remote wal checkpoint intervals

Signed-off-by: WenyXu <wenymedia@gmail.com>

* feat: add logical pruning fuzz target

Signed-off-by: WenyXu <wenymedia@gmail.com>

* ci: add remote WAL logical pruning fuzz

Signed-off-by: WenyXu <wenymedia@gmail.com>

* chore: apply suggestions

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>

.github/actions/setup-greptimedb-cluster/kafka-wal-helper.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-wal-helper
+  labels:
+    app: kafka-wal-helper
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-wal-helper
+  template:
+    metadata:
+      labels:
+        app: kafka-wal-helper
+    spec:
+      containers:
+        - name: kafka-wal-helper
+          image: localhost:5001/greptime/kafka-wal-helper:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--addr"
+            - "0.0.0.0:8080"
+          ports:
+            - name: http
+              containerPort: 8080
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-wal-helper
+  labels:
+    app: kafka-wal-helper
+spec:
+  type: ClusterIP
+  selector:
+    app: kafka-wal-helper
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml

@@ -0,0 +1,63 @@
+logging:
+  level: "info"
+  format: "json"
+  filters:
+  - log_store=debug
+meta:
+  configData: |-
+    [runtime]
+    global_rt_size = 4
+
+    [wal]
+    provider = "kafka"
+    broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
+    num_topics = 3
+    auto_prune_interval = "30s"
+    auto_prune_parallelism = 3
+    auto_prune_logical_delete = true
+    trigger_flush_threshold = 100
+    region_flush_trigger_interval = "30s"
+    periodic_checkpoint_persist_interval = "60s"
+    flush_trigger_size = "1MB"
+    checkpoint_trigger_size = "1MB"
+
+    [datanode]
+    [datanode.client]
+    timeout = "120s"
+datanode:
+  configData: |-
+    [runtime]
+    global_rt_size = 4
+    compact_rt_size = 2
+
+    [wal]
+    provider = "kafka"
+    broker_endpoints = ["kafka.kafka-cluster.svc.cluster.local:9092"]
+    num_topics = 3
+    topic_latest_offset_fetch_interval = "30s"
+    overwrite_entry_start_id = false
+
+    [[region_engine]]
+    [region_engine.mito]
+    auto_flush_interval = "30s"
+frontend:
+  configData: |-
+    [runtime]
+    global_rt_size = 4
+
+    [meta_client]
+    ddl_timeout = "120s"
+objectStorage:
+  s3:
+    bucket: default
+    region: us-west-2
+    root: test-root
+    endpoint: http://minio.minio.svc.cluster.local
+  credentials:
+    accessKeyId: rootuser
+    secretAccessKey: rootpass123
+remoteWal:
+   enabled: true
+   kafka:
+     brokerEndpoints:
+      - "kafka.kafka-cluster.svc.cluster.local:9092"

.github/workflows/develop.yml

@@ -329,6 +329,13 @@ jobs:
               minio: true
               kafka: false
               values: "with-minio-repartition-gc.yaml"
+          - target: "fuzz_remote_wal_logical_prune"
+            mode:
+              name: "Remote WAL logical pruning"
+              minio: true
+              kafka: true
+              values: "with-remote-wal-logical-prune.yaml"
+              kafkaWalHelper: true
     steps:
       - name: Remove unused software
         run: |
@@ -378,6 +385,24 @@ jobs:
           rm ./bin.tar.gz
       - name: Build and push GreptimeDB image
         uses: ./.github/actions/build-and-push-ci-image
+      - if: matrix.mode.kafkaWalHelper
+        name: Build Kafka WAL helper binary
+        shell: bash
+        run: |
+          cargo build --bin kafka_wal_helper
+          mkdir -p amd64
+          cp target/debug/kafka_wal_helper amd64/kafka_wal_helper
+      - if: matrix.mode.kafkaWalHelper
+        name: Build and push Kafka WAL helper image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/ci/ubuntu/Dockerfile.kafka-wal-helper
+          push: true
+          tags: localhost:5001/greptime/kafka-wal-helper:latest
+          build-args: |
+            TARGETARCH=amd64
+            TARGET_BIN=kafka_wal_helper
       - name: Wait for etcd
         run: |
           kubectl wait \
@@ -401,6 +426,17 @@ jobs:
             pod -l app.kubernetes.io/instance=kafka \
             --timeout=120s \
             -n kafka-cluster
+      - if: matrix.mode.kafkaWalHelper
+        name: Setup Kafka WAL helper
+        shell: bash
+        run: |
+          kubectl -n kafka-cluster apply \
+            -f .github/actions/setup-greptimedb-cluster/kafka-wal-helper.yaml
+
+          kubectl -n kafka-cluster wait \
+            --for=condition=Ready \
+            pod -l app=kafka-wal-helper \
+            --timeout=120s
       - name: Print etcd info
         shell: bash
         run: kubectl get all --show-labels -n etcd-cluster
@@ -413,14 +449,23 @@ jobs:
       - name: Port forward (mysql)
         run: |
           kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
+      - if: matrix.mode.kafkaWalHelper
+        name: Port forward Kafka WAL helper
+        run: |
+          kubectl port-forward service/kafka-wal-helper 8080:8080 -n kafka-cluster&
       - name: Fuzz Test
         uses: ./.github/actions/fuzz-test
         env:
           CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
           GT_MYSQL_ADDR: 127.0.0.1:4002
+          GT_KAFKA_WAL_HELPER_URL: http://127.0.0.1:8080
+          GT_KAFKA_ADDR: kafka.kafka-cluster.svc.cluster.local:9092
+          GT_WAL_TOPIC_PREFIX: greptimedb_wal_topic
+          GT_WAL_NUM_TOPICS: 3
+          GT_FUZZ_OVERRIDE_DELETE_AFTER_SECS: 300
         with:
           target: ${{ matrix.target }}
-          max-total-time: 120
+          max-total-time: ${{ matrix.mode.kafkaWalHelper && 600 || 120 }}
       - name: Describe Nodes
         if: failure()
         shell: bash