mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-21 10:14:06 +00:00
ci: compare GreptimeDB File and S3 in observability benchmarks (#10187)
* ci: compare GreptimeDB File and S3 in observability benchmarks Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: separate File and S3 benchmark steps and verify report artifacts Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
@@ -2,6 +2,11 @@ name: Agent Observability Benchmark
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -56,7 +61,7 @@ on:
|
||||
type: string
|
||||
required: false
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -99,6 +104,11 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
PROFILE: ${{ inputs.dataset }}
|
||||
@@ -106,7 +116,7 @@ env:
|
||||
CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
|
||||
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
|
||||
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -130,6 +140,10 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -370,24 +384,48 @@ jobs:
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
|
||||
cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
|
||||
- name: Load and benchmark baseline GreptimeDB
|
||||
id: baseline_greptimedb
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
|
||||
- name: Load and benchmark baseline GreptimeDB (File)
|
||||
id: baseline_greptimedb_file
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
|
||||
--owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb
|
||||
id: greptimedb
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
|
||||
- name: Load and benchmark baseline GreptimeDB (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
id: baseline_greptimedb_s3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
|
||||
--owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline/s3" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb (File)
|
||||
id: greptimedb_file
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
id: greptimedb_s3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
|
||||
--owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/s3" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark clickhouse
|
||||
id: clickhouse
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }}
|
||||
@@ -407,7 +445,7 @@ jobs:
|
||||
--target victorialogs --image "$VICTORIALOGS_IMAGE" \
|
||||
--victorialogs-http-port "$VICTORIALOGS_HTTP_PORT"
|
||||
- name: Compare all three targets
|
||||
if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
|
||||
if: ${{ !cancelled() && steps.greptimedb_file.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
|
||||
== 'success' }}
|
||||
run: |-
|
||||
docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
|
||||
@@ -418,45 +456,61 @@ jobs:
|
||||
--victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \
|
||||
--output /results/comparison.json
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always()
|
||||
run: |-
|
||||
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
|
||||
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--owner "$OWNER" --root "$ARTIFACT_ROOT"
|
||||
fi
|
||||
- name: Clean up baseline containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |-
|
||||
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
|
||||
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
|
||||
fi
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Load / lifecycle summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Baseline load / lifecycle summary
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '## Baseline GreptimeDB' | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --agent "$ARTIFACT_ROOT" \
|
||||
--output "$ARTIFACT_ROOT-query-report.html" --json-output "$ARTIFACT_ROOT-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-query-report.json" \
|
||||
| tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT" \
|
||||
--report "$ARTIFACT_ROOT-query-report.json" --targets "$TARGETS"
|
||||
- name: Upload results (no dataset)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |-
|
||||
${{ env.ARTIFACT_ROOT }}-query-report.*
|
||||
${{ env.MANIFEST_PATH }}
|
||||
${{ env.ARTIFACT_ROOT }}
|
||||
!${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl
|
||||
|
||||
@@ -2,6 +2,11 @@ name: Tracesbench
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -39,7 +44,7 @@ on:
|
||||
default: sanity
|
||||
required: true
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -107,6 +112,11 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
DATASET: ${{ inputs.dataset }}
|
||||
@@ -114,7 +124,7 @@ env:
|
||||
RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
|
||||
RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
|
||||
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -162,6 +172,10 @@ jobs:
|
||||
[[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
|
||||
fi
|
||||
printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -389,22 +403,44 @@ jobs:
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark (File)
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/baseline/s3" --data-root "$RUN_ROOT" --owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/s3/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark (File)
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/s3/runs/greptimedb"
|
||||
- name: VictoriaTraces load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
|
||||
run: |
|
||||
@@ -422,49 +458,80 @@ jobs:
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Validate and render report
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Validate and render primary report
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
run: |
|
||||
if [[ "$GREPTIMEDB_STORAGE" == s3 ]]; then
|
||||
TARGETS=$(tr "," "\n" <<< "$TARGETS" | sed "/^greptimedb$/d" | paste -sd, -)
|
||||
[[ -n "$TARGETS" ]] || exit 0
|
||||
fi
|
||||
comparison_args=()
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true && "$GREPTIMEDB_STORAGE" != s3 ]]; then
|
||||
comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
|
||||
fi
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "${TARGETS}" "${comparison_args[@]}" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Validate and render S3 report
|
||||
if: always() && steps.generate.outcome == 'success' && inputs.greptimedb_storage != 'file'
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets greptimedb \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && steps.runtime.outcome == 'success'
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up baseline containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
--root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
|
||||
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Load / lifecycle summary
|
||||
if: always()
|
||||
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Baseline load / lifecycle summary
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/tracesbench-summary.py \
|
||||
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '## Baseline GreptimeDB' | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces \
|
||||
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
mkdir -p "$RUN_ROOT/metadata"
|
||||
cp "$METADATA_ROOT"/*-image.json "$RUN_ROOT/metadata/"
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --traces "$RUN_ROOT" \
|
||||
--output "$RUN_ROOT-query-report.html" --json-output "$RUN_ROOT-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$RUN_ROOT-query-report.json" \
|
||||
| tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$RUN_ROOT" \
|
||||
--report "$RUN_ROOT-query-report.json" --targets "$TARGETS"
|
||||
- name: Stage benchmark artifacts
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts"
|
||||
cp "$RUN_ROOT"-query-report.* "$RUNNER_TEMP/tracesbench-artifacts/" 2>/dev/null || true
|
||||
if [[ -d "$METADATA_ROOT" ]]; then
|
||||
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
|
||||
fi
|
||||
for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
|
||||
for entry in artifact-audit.json metadata runs report report.json report.md ci-summary.md version-comparison.json version-comparison.md; do
|
||||
if [[ -e "$RUN_ROOT/$entry" ]]; then
|
||||
rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
|
||||
fi
|
||||
@@ -473,6 +540,12 @@ jobs:
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
|
||||
rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
|
||||
fi
|
||||
for arm in s3 baseline/s3; do
|
||||
if [[ -d "$RUN_ROOT/$arm" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/$arm"
|
||||
rsync -a --no-owner --no-group --exclude scratch --exclude corpora "$RUN_ROOT/$arm/" "$RUNNER_TEMP/tracesbench-artifacts/$arm/"
|
||||
fi
|
||||
done
|
||||
- name: Upload HTML and results (no corpus)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -2,6 +2,11 @@ name: VMBench Long Range
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -41,7 +46,7 @@ on:
|
||||
default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
|
||||
required: true
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -89,13 +94,18 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
|
||||
RANGE_PRESET: ${{ inputs.range_preset }}
|
||||
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
|
||||
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -119,6 +129,10 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -173,7 +187,7 @@ jobs:
|
||||
[[ "$revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
|
||||
printf 'run_id=long-range-%s-%s\nplan_artifact=vmbench-plan-%s-%s\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
|
||||
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
|
||||
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --greptimedb-storage "$GREPTIMEDB_STORAGE"
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
|
||||
--run-classification controlled_server_benchmark --no-open-browser)
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
|
||||
@@ -192,10 +206,11 @@ jobs:
|
||||
# The packaged planner owns profile, target and resource-limit validation.
|
||||
mkdir -p "$RUNNER_TEMP/vmbench-plans"
|
||||
# Keep the original warm defaults: serial 5/1 and c1/c4 30s/5s.
|
||||
docker run --rm "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
|
||||
docker run --rm -e S3_BUCKET -e S3_REGION "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
|
||||
--run-id "long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
|
||||
--output-root "$BENCHMARK_ROOT/o11ybench/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
|
||||
--cache-policy warm > "$RUNNER_TEMP/vmbench-plans/warm.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py normalize-metrics "$RUNNER_TEMP/vmbench-plans/warm.json"
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: vmbench-plan-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
@@ -340,25 +355,87 @@ jobs:
|
||||
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --dry-run \
|
||||
> "$METADATA_ROOT/warm-execution-plan.json"
|
||||
cat "$METADATA_ROOT/warm-execution-plan.json"
|
||||
- name: Warm serial and concurrent benchmarks
|
||||
- name: Generate shared dataset and split target plans
|
||||
id: generate
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -u o11ybench/scripts/run_benchmark_plan.py \
|
||||
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --execute \
|
||||
2>&1 | tee "$METADATA_ROOT/warm-execution.log"
|
||||
python3 o11ybench/scripts/check_run_benchmark_plan.py --artifacts "$ARTIFACT_ROOT-warm"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-prepare \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark Baseline GreptimeDB File
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-baseline \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark Baseline GreptimeDB S3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 'file' }}
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3-baseline \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark GreptimeDB File
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark GreptimeDB S3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 'file' }}
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3 \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark VictoriaMetrics
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_victoriametrics }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target victoriametrics \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Validate all selected target results
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-validate \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always()
|
||||
run: |
|
||||
docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
|
||||
for name in greptimedb greptimedb-s3 greptimedb-baseline greptimedb-s3-baseline victoriametrics; do
|
||||
plan="$METADATA_ROOT/$name.json"
|
||||
[[ -f "$plan" ]] || continue
|
||||
run_id=$(jq -r '.run_id // empty' "$plan")
|
||||
[[ -n "$run_id" ]] || continue
|
||||
docker ps -aq --filter "label=o11ybench.run-id=$run_id" | xargs -r docker rm -fv
|
||||
done
|
||||
if [[ -d "$ARTIFACT_ROOT-warm" ]]; then
|
||||
for probe in "$ARTIFACT_ROOT-warm"/o11ybench/*/*/s3-probe.json; do
|
||||
[[ -f "$probe" ]] || continue
|
||||
python3 o11ybench/scripts/greptimedb_storage.py recover --runtime-image "$RESOLVED_RUNTIME_IMAGE" --output "${probe%/*}"
|
||||
done
|
||||
fi
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" | tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
mkdir -p "$ARTIFACT_ROOT-warm/metadata"
|
||||
cp "$METADATA_ROOT"/*-image.json "$ARTIFACT_ROOT-warm/metadata/"
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --metrics "$ARTIFACT_ROOT-warm" \
|
||||
--output "$ARTIFACT_ROOT-warm-query-report.html" --json-output "$ARTIFACT_ROOT-warm-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-warm-query-report.json" \
|
||||
| tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT-warm" \
|
||||
--report "$ARTIFACT_ROOT-warm-query-report.json" --targets "$(jq -r '[.targets[].kind] | unique | join(",")' "$METADATA_ROOT/warm.json")"
|
||||
- name: Stage benchmark artifacts
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/vmbench-artifacts"
|
||||
cp "$ARTIFACT_ROOT-warm"-query-report.* "$RUNNER_TEMP/vmbench-artifacts/" 2>/dev/null || true
|
||||
if [[ -d "$METADATA_ROOT" ]]; then
|
||||
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/vmbench-artifacts/"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user