chore(ci): refactor

2026-05-20 06:50:37 +00:00 · 2024-06-05 06:44:39 +00:00
parent 67205cd3d8
commit a358d03785
5 changed files with 63 additions and 118 deletions
--- a/.github/actions/setup-greptimedb-cluster/action.yml
+++ b/.github/actions/setup-greptimedb-cluster/action.yml
@@ -23,7 +23,7 @@ inputs:
    default: "etcd.etcd-cluster.svc.cluster.local:2379"
    description: "Etcd endpoints"
  values-filename:
-    default: "values.yaml"
+    default: "with-minio.yaml"
    

 runs:
@@ -57,12 +57,6 @@ runs:
        --set frontend.replicas=${{ inputs.frontend-replicas }} \
        --set datanode.replicas=${{ inputs.datanode-replicas }} \
        --set meta.replicas=${{ inputs.meta-replicas }} \
-        --set objectStorage.s3.bucket=default \
-        --set objectStorage.s3.region=us-west-2 \
-        --set objectStorage.s3.root=test-root \
-        --set objectStorage.s3.endpoint=http://minio.minio.svc.cluster.local \
-        --set objectStorage.credentials.accessKeyId=rootuser \
-        --set objectStorage.credentials.secretAccessKey=rootpass123 \
        greptime/greptimedb-cluster \
        --create-namespace \
        -n my-greptimedb \
--- a/.github/actions/setup-greptimedb-cluster/with-disk.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-disk.yaml
--- a/.github/actions/setup-greptimedb-cluster/with-minio.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-minio.yaml
@@ -0,0 +1,27 @@
+meta:
+  config: |-
+    [runtime]
+    read_rt_size = 8
+    write_rt_size = 8
+    bg_rt_size = 8
+datanode:
+  config: |-
+    [runtime]
+    read_rt_size = 8
+    write_rt_size = 8
+    bg_rt_size = 8
+frontend:
+  config: |-
+    [runtime]
+    read_rt_size = 8
+    write_rt_size = 8
+    bg_rt_size = 8
+objectStorage:
+  s3:
+    bucket: default
+    region: us-west-2
+    root: test-root
+    endpoint: http://minio.minio.svc.cluster.local 
+  credentials:
+    accessKeyId: rootuser
+    secretAccessKey: rootpass123
--- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml
@@ -26,3 +26,12 @@ frontend:
    read_rt_size = 8
    write_rt_size = 8
    bg_rt_size = 8
+objectStorage:
+  s3:
+    bucket: default
+    region: us-west-2
+    root: test-root
+    endpoint: http://minio.minio.svc.cluster.local 
+  credentials:
+    accessKeyId: rootuser
+    secretAccessKey: rootpass123
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -277,16 +277,35 @@ jobs:
          version: current

  distributed-fuzztest: 
-    name: Fuzz Test (Distributed, Minio)
+    name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
    runs-on: ubuntu-latest
    needs:  build-greptime-ci
    strategy:
      matrix:
        target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
+        mode: 
+          - name: "Disk"
+            minio: false
+            kafka: false
+            values: "with-disk.yaml"
+          - name: "Minio"
+            minio: true
+            kafka: false
+            values: "with-minio.yaml"
+          - name: "Remote WAL"
+            minio: true
+            kafka: true
+            values: "with-remote-wal.yaml"
    steps:
      - uses: actions/checkout@v4
      - name: Setup Kind
        uses: ./.github/actions/setup-kind
+      - if: matrix.mode.minio
+        name: Setup Minio
+        uses: ./.github/actions/setup-minio
+      - if: matrix.mode.kafka
+        name: Setup Kafka cluser
+        uses: ./.github/actions/setup-kafka-cluster
      - name: Setup Etcd cluser
        uses: ./.github/actions/setup-etcd-cluster
      - name: Setup Minio
@@ -329,126 +348,22 @@ jobs:
            pod -l app.kubernetes.io/instance=etcd \
            --timeout=120s \
            -n etcd-cluster
-      - name: Wait for minio
+      - if: matrix.mode.minio
+        name: Wait for minio
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app=minio \
            --timeout=120s \
            -n minio
-      - name: Print etcd info
-        shell: bash
-        run: kubectl get all --show-labels -n etcd-cluster
-      # Setup cluster for test
-      - name: Setup GreptimeDB cluster
-        uses: ./.github/actions/setup-greptimedb-cluster
-        with:
-          image-registry: localhost:5001
-      - name: Port forward (mysql)
-        run: |
-          kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
-      - name: Fuzz Test
-        uses: ./.github/actions/fuzz-test
-        env:
-          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
-          GT_MYSQL_ADDR: 127.0.0.1:4002
-        with:
-          target: ${{ matrix.target }}
-          max-total-time: 120
-      - name: Describe Nodes
-        if: failure()
-        shell: bash
-        run: | 
-          kubectl describe nodes      
-      - name: Export kind logs
-        if: failure()
-        shell: bash
-        run: | 
-          kind export logs /tmp/kind
-      - name: Upload logs
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: fuzz-tests-kind-logs-${{ matrix.target }}
-          path: /tmp/kind
-          retention-days: 3
-      - name: Delete cluster
-        if: success()
-        shell: bash
-        run: | 
-          kind delete cluster
-          docker stop $(docker ps -a -q)
-          docker rm $(docker ps -a -q)
-          docker system prune -f
-
-  distributed-fuzztest-with-kafka: 
-    name: Fuzz Test with Kafka (Distributed, Minio)
-    runs-on: ubuntu-latest
-    needs:  build-greptime-ci
-    strategy:
-      matrix:
-        target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Kind
-        uses: ./.github/actions/setup-kind
-      - name: Setup Etcd cluser
-        uses: ./.github/actions/setup-etcd-cluster
-      - name: Setup Minio
-        uses: ./.github/actions/setup-minio
-      - name: Setup Kafka cluser
-        uses: ./.github/actions/setup-kafka-cluster
-      # Prepares for fuzz tests
-      - uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_TOOLCHAIN }}
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "fuzz-test-targets"
-      - name: Set Rust Fuzz
-        shell: bash
-        run: |
-          sudo apt-get install -y libfuzzer-14-dev
-          rustup install nightly
-          cargo +nightly install cargo-fuzz
-      # Downloads ci image
-      - name: Download pre-built binariy
-        uses: actions/download-artifact@v4
-        with:
-          name: bin
-          path: .
-      - name: Unzip binary
-        run: |
-          tar -xvf ./bin.tar.gz
-          rm ./bin.tar.gz
-      - name: Build and push GreptimeDB image
-        uses: ./.github/actions/build-and-push-ci-image
-      - name: Wait for etcd
-        run: |
-          kubectl wait \
-            --for=condition=Ready \
-            pod -l app.kubernetes.io/instance=etcd \
-            --timeout=120s \
-            -n etcd-cluster
-      - name: Wait for kafka
+      - if: matrix.mode.kafka
+        name: Wait for kafka
        run: |
          kubectl wait \
            --for=condition=Ready \
            pod -l app.kubernetes.io/instance=kafka \
            --timeout=120s \
            -n kafka-cluster
-      - name: Wait for minio
-        run: |
-          kubectl wait \
-            --for=condition=Ready \
-            pod -l app=minio \
-            --timeout=120s \
-            -n minio
      - name: Print etcd info
        shell: bash
        run: kubectl get all --show-labels -n etcd-cluster
@@ -457,7 +372,7 @@ jobs:
        uses: ./.github/actions/setup-greptimedb-cluster
        with:
          image-registry: localhost:5001
-          values-filename:  "kafka-values.yaml"
+          values-filename: ${{ matrix.mode.values }}
      - name: Port forward (mysql)
        run: |
          kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
@@ -483,7 +398,7 @@ jobs:
        if: failure()
        uses: actions/upload-artifact@v4
        with:
-          name: fuzz-tests-with-kafka-kind-logs-${{ matrix.target }}
+          name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
          path: /tmp/kind
          retention-days: 3
      - name: Delete cluster