ci: compare GreptimeDB File and S3 in observability benchmarks (#10187)

* ci: compare GreptimeDB File and S3 in observability benchmarks

Signed-off-by: WenyXu <wenymedia@gmail.com>

* ci: separate File and S3 benchmark steps and verify report artifacts

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
2026-11-05 18:19:56 +00:00
committed by GitHub
parent 4263f5fc01
commit 5bcb576b49
8 changed files with 696 additions and 71 deletions
+75 -21
View File
@@ -2,6 +2,11 @@ name: Agent Observability Benchmark
on:
workflow_dispatch:
inputs:
greptimedb_storage:
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
type: choice
options: [file, s3, both]
default: file
run_greptimedb:
description: Run GreptimeDB
type: boolean
@@ -56,7 +61,7 @@ on:
type: string
required: false
runtime_image:
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
type: string
default: auto
required: true
@@ -99,6 +104,11 @@ on:
permissions:
contents: read
env:
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
PROFILE: ${{ inputs.dataset }}
@@ -106,7 +116,7 @@ env:
CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -130,6 +140,10 @@ jobs:
with:
path: greptimedb
persist-credentials: false
- name: Validate GreptimeDB storage
run: |
python3 greptimedb/tests/perf/test_o11ybench_storage.py
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
- name: Validate version comparison
env:
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
@@ -370,24 +384,48 @@ jobs:
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
--root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
- name: Load and benchmark baseline GreptimeDB
id: baseline_greptimedb
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
- name: Load and benchmark baseline GreptimeDB (File)
id: baseline_greptimedb_file
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
run: |-
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
--owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
- name: Load and benchmark greptimedb
id: greptimedb
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
- name: Load and benchmark baseline GreptimeDB (S3)
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
id: baseline_greptimedb_s3
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
run: |-
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
--owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline/s3" --data-root "$ARTIFACT_ROOT" \
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
- name: Load and benchmark greptimedb (File)
id: greptimedb_file
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
run: |-
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
- name: Load and benchmark greptimedb (S3)
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
id: greptimedb_s3
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
run: |-
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
--owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/s3" --data-root "$ARTIFACT_ROOT" \
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
- name: Load and benchmark clickhouse
id: clickhouse
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }}
@@ -407,7 +445,7 @@ jobs:
--target victorialogs --image "$VICTORIALOGS_IMAGE" \
--victorialogs-http-port "$VICTORIALOGS_HTTP_PORT"
- name: Compare all three targets
if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
if: ${{ !cancelled() && steps.greptimedb_file.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
== 'success' }}
run: |-
docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
@@ -418,45 +456,61 @@ jobs:
--victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \
--output /results/comparison.json
- name: Clean up owned containers
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: always()
run: |-
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
--owner "$OWNER" --root "$ARTIFACT_ROOT"
fi
- name: Clean up baseline containers
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: always() && inputs.compare_greptimedb
run: |-
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
--owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
fi
- name: Benchmark summary
if: always()
run: |
python3 greptimedb/.github/scripts/agent-observability-summary.py \
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Load / lifecycle summary
if: always()
run: |
python3 greptimedb/.github/scripts/agent-observability-summary.py \
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Baseline load / lifecycle summary
if: always() && inputs.compare_greptimedb
run: |
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/agent-observability-summary.py \
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
echo '## Baseline GreptimeDB' | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: GreptimeDB version comparison
if: always() && inputs.compare_greptimedb
run: |
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Render storage comparison overview
if: always() && steps.generate.outcome == 'success'
run: |
python3 o11ybench/scripts/build_query_performance_report.py --agent "$ARTIFACT_ROOT" \
--output "$ARTIFACT_ROOT-query-report.html" --json-output "$ARTIFACT_ROOT-query-report.json"
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-query-report.json" \
| tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT" \
--report "$ARTIFACT_ROOT-query-report.json" --targets "$TARGETS"
- name: Upload results (no dataset)
if: always()
uses: actions/upload-artifact@v4
with:
name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }}
path: |-
${{ env.ARTIFACT_ROOT }}-query-report.*
${{ env.MANIFEST_PATH }}
${{ env.ARTIFACT_ROOT }}
!${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl
+93 -20
View File
@@ -2,6 +2,11 @@ name: Tracesbench
on:
workflow_dispatch:
inputs:
greptimedb_storage:
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
type: choice
options: [file, s3, both]
default: file
run_greptimedb:
description: Run GreptimeDB
type: boolean
@@ -39,7 +44,7 @@ on:
default: sanity
required: true
runtime_image:
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
type: string
default: auto
required: true
@@ -107,6 +112,11 @@ on:
permissions:
contents: read
env:
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
DATASET: ${{ inputs.dataset }}
@@ -114,7 +124,7 @@ env:
RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -162,6 +172,10 @@ jobs:
[[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
fi
printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
- name: Validate GreptimeDB storage
run: |
python3 greptimedb/tests/perf/test_o11ybench_storage.py
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
- name: Validate version comparison
env:
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
@@ -389,22 +403,44 @@ jobs:
python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Baseline GreptimeDB load, correctness and benchmark
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
- name: Baseline GreptimeDB load, correctness and benchmark (File)
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
- name: GreptimeDB load, correctness and benchmark
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
- name: Baseline GreptimeDB load, correctness and benchmark (S3)
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT/baseline/s3" --data-root "$RUN_ROOT" --owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/s3/runs/greptimedb"
- name: GreptimeDB load, correctness and benchmark (File)
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
- name: GreptimeDB load, correctness and benchmark (S3)
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/s3/runs/greptimedb"
- name: VictoriaTraces load, correctness and benchmark
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
run: |
@@ -422,49 +458,80 @@ jobs:
- name: GreptimeDB version comparison
if: always() && inputs.compare_greptimedb
run: |
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
- name: Validate and render report
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Validate and render primary report
if: always() && steps.generate.outcome == 'success'
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
run: |
if [[ "$GREPTIMEDB_STORAGE" == s3 ]]; then
TARGETS=$(tr "," "\n" <<< "$TARGETS" | sed "/^greptimedb$/d" | paste -sd, -)
[[ -n "$TARGETS" ]] || exit 0
fi
comparison_args=()
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
if [[ "$COMPARE_GREPTIMEDB" == true && "$GREPTIMEDB_STORAGE" != s3 ]]; then
comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
fi
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "${TARGETS}" "${comparison_args[@]}" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Validate and render S3 report
if: always() && steps.generate.outcome == 'success' && inputs.greptimedb_storage != 'file'
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets greptimedb \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Clean up owned containers
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: always() && steps.runtime.outcome == 'success'
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Clean up baseline containers
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
--root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Benchmark summary
if: always()
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Load / lifecycle summary
if: always()
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Baseline load / lifecycle summary
if: always() && inputs.compare_greptimedb
run: |
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/tracesbench-summary.py \
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
echo '## Baseline GreptimeDB' | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces \
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Render storage comparison overview
if: always() && steps.generate.outcome == 'success'
run: |
mkdir -p "$RUN_ROOT/metadata"
cp "$METADATA_ROOT"/*-image.json "$RUN_ROOT/metadata/"
python3 o11ybench/scripts/build_query_performance_report.py --traces "$RUN_ROOT" \
--output "$RUN_ROOT-query-report.html" --json-output "$RUN_ROOT-query-report.json"
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$RUN_ROOT-query-report.json" \
| tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$RUN_ROOT" \
--report "$RUN_ROOT-query-report.json" --targets "$TARGETS"
- name: Stage benchmark artifacts
if: always()
run: |
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts"
cp "$RUN_ROOT"-query-report.* "$RUNNER_TEMP/tracesbench-artifacts/" 2>/dev/null || true
if [[ -d "$METADATA_ROOT" ]]; then
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
fi
for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
for entry in artifact-audit.json metadata runs report report.json report.md ci-summary.md version-comparison.json version-comparison.md; do
if [[ -e "$RUN_ROOT/$entry" ]]; then
rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
fi
@@ -473,6 +540,12 @@ jobs:
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
fi
for arm in s3 baseline/s3; do
if [[ -d "$RUN_ROOT/$arm" ]]; then
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/$arm"
rsync -a --no-owner --no-group --exclude scratch --exclude corpora "$RUN_ROOT/$arm/" "$RUNNER_TEMP/tracesbench-artifacts/$arm/"
fi
done
- name: Upload HTML and results (no corpus)
if: always()
uses: actions/upload-artifact@v4
+88 -11
View File
@@ -2,6 +2,11 @@ name: VMBench Long Range
on:
workflow_dispatch:
inputs:
greptimedb_storage:
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
type: choice
options: [file, s3, both]
default: file
run_greptimedb:
description: Run GreptimeDB
type: boolean
@@ -41,7 +46,7 @@ on:
default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
required: true
runtime_image:
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
type: string
default: auto
required: true
@@ -89,13 +94,18 @@ on:
permissions:
contents: read
env:
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
RANGE_PRESET: ${{ inputs.range_preset }}
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -119,6 +129,10 @@ jobs:
with:
path: greptimedb
persist-credentials: false
- name: Validate GreptimeDB storage
run: |
python3 greptimedb/tests/perf/test_o11ybench_storage.py
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
- name: Validate version comparison
env:
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
@@ -173,7 +187,7 @@ jobs:
[[ "$revision" =~ ^[0-9a-f]{40}$ ]]
printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
printf 'run_id=long-range-%s-%s\nplan_artifact=vmbench-plan-%s-%s\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --greptimedb-storage "$GREPTIMEDB_STORAGE"
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
--run-classification controlled_server_benchmark --no-open-browser)
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
@@ -192,10 +206,11 @@ jobs:
# The packaged planner owns profile, target and resource-limit validation.
mkdir -p "$RUNNER_TEMP/vmbench-plans"
# Keep the original warm defaults: serial 5/1 and c1/c4 30s/5s.
docker run --rm "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
docker run --rm -e S3_BUCKET -e S3_REGION "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
--run-id "long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
--output-root "$BENCHMARK_ROOT/o11ybench/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
--cache-policy warm > "$RUNNER_TEMP/vmbench-plans/warm.json"
python3 greptimedb/.github/scripts/o11ybench-storage.py normalize-metrics "$RUNNER_TEMP/vmbench-plans/warm.json"
- uses: actions/upload-artifact@v4
with:
name: vmbench-plan-${{ github.run_id }}-${{ github.run_attempt }}
@@ -340,25 +355,87 @@ jobs:
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --dry-run \
> "$METADATA_ROOT/warm-execution-plan.json"
cat "$METADATA_ROOT/warm-execution-plan.json"
- name: Warm serial and concurrent benchmarks
- name: Generate shared dataset and split target plans
id: generate
run: |
set -euo pipefail
python3 -u o11ybench/scripts/run_benchmark_plan.py \
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --execute \
2>&1 | tee "$METADATA_ROOT/warm-execution.log"
python3 o11ybench/scripts/check_run_benchmark_plan.py --artifacts "$ARTIFACT_ROOT-warm"
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-prepare \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Load and benchmark Baseline GreptimeDB File
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 's3' }}
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-baseline \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Load and benchmark Baseline GreptimeDB S3
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 'file' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3-baseline \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Load and benchmark GreptimeDB File
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 's3' }}
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Load and benchmark GreptimeDB S3
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 'file' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3 \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Load and benchmark VictoriaMetrics
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_victoriametrics }}
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target victoriametrics \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Validate all selected target results
if: always() && steps.generate.outcome == 'success'
run: |
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-validate \
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Clean up owned containers
env:
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
if: always()
run: |
docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
for name in greptimedb greptimedb-s3 greptimedb-baseline greptimedb-s3-baseline victoriametrics; do
plan="$METADATA_ROOT/$name.json"
[[ -f "$plan" ]] || continue
run_id=$(jq -r '.run_id // empty' "$plan")
[[ -n "$run_id" ]] || continue
docker ps -aq --filter "label=o11ybench.run-id=$run_id" | xargs -r docker rm -fv
done
if [[ -d "$ARTIFACT_ROOT-warm" ]]; then
for probe in "$ARTIFACT_ROOT-warm"/o11ybench/*/*/s3-probe.json; do
[[ -f "$probe" ]] || continue
python3 o11ybench/scripts/greptimedb_storage.py recover --runtime-image "$RESOLVED_RUNTIME_IMAGE" --output "${probe%/*}"
done
fi
- name: GreptimeDB version comparison
if: always() && inputs.compare_greptimedb
run: |
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" | tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
- name: Render storage comparison overview
if: always() && steps.generate.outcome == 'success'
run: |
mkdir -p "$ARTIFACT_ROOT-warm/metadata"
cp "$METADATA_ROOT"/*-image.json "$ARTIFACT_ROOT-warm/metadata/"
python3 o11ybench/scripts/build_query_performance_report.py --metrics "$ARTIFACT_ROOT-warm" \
--output "$ARTIFACT_ROOT-warm-query-report.html" --json-output "$ARTIFACT_ROOT-warm-query-report.json"
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-warm-query-report.json" \
| tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT-warm" \
--report "$ARTIFACT_ROOT-warm-query-report.json" --targets "$(jq -r '[.targets[].kind] | unique | join(",")' "$METADATA_ROOT/warm.json")"
- name: Stage benchmark artifacts
if: always()
run: |
mkdir -p "$RUNNER_TEMP/vmbench-artifacts"
cp "$ARTIFACT_ROOT-warm"-query-report.* "$RUNNER_TEMP/vmbench-artifacts/" 2>/dev/null || true
if [[ -d "$METADATA_ROOT" ]]; then
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/vmbench-artifacts/"
fi