rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-12 23:12:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: add optional AWS runners for observability benchmarks

Browse Source 
Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
WenyXu
2026-09-23 08:39:23 +00:00
parent a35f9d5c78
commit 1103473425
14 changed files with 1288 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/agent-observability.yml
+72 -19
View File
@@ -50,15 +50,21 @@ on:
type: string
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0
required: true
ecs_instance_type:
description: ECS instance type (independent of Query Regression)
provider:
description: Cloud provider for the benchmark runner
type: choice
options: [Aliyun, AWS]
default: Aliyun
required: true
instance_type:
description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
type: string
default: ecs.c9i.2xlarge
default: auto
required: true
system_disk_gib:
description: System disk GiB; 0 = auto (S 80, P 100, M 500), or 20..2048
description: System disk GiB (default 500; 0 uses default)
type: number
default: 0
default: 500
required: true
benchmark_timeout_minutes:
description: Benchmark job timeout in minutes (1..360)
@@ -89,7 +95,8 @@ env:
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
RUNTIME_IMAGE: ${{ inputs.runtime_image }}
ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
@@ -100,6 +107,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
instance_type: ${{ steps.runner.outputs.instance_type }}
targets: ${{ steps.inputs.outputs.targets }}
system_disk_gib: ${{ steps.inputs.outputs.system_disk_gib }}
greptimedb: ${{ steps.inputs.outputs.greptimedb }}
@@ -110,9 +118,13 @@ jobs:
with:
path: greptimedb
persist-credentials: false
- name: Test ECS lifecycle helpers
- name: Resolve CI runner
id: runner
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
- name: Test CI runner helpers
run: |
python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
python3 greptimedb/tests/perf/test_ci_runner_scripts.py
python3 greptimedb/tests/perf/test_agent_observability_summary.py
- name: Validate requested benchmark
id: inputs
@@ -124,14 +136,11 @@ jobs:
run: |
set -euo pipefail
# Validate cloud lifecycle budgets before provisioning; runtime owns workload validation.
[[ -n "$ECS_INSTANCE_TYPE" ]]
case "$PROFILE" in
S) auto_disk_gib=80 ;;
P) auto_disk_gib=100 ;;
M) auto_disk_gib=500 ;;
S|P|M) ;;
*) echo "Unknown dataset: $PROFILE" >&2; exit 1 ;;
esac
[[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=$auto_disk_gib
[[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=500
for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
[[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; }
done
@@ -160,15 +169,16 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
outputs:
label: ${{ steps.ecs.outputs.label }}
instance_id: ${{ steps.ecs.outputs.instance_id }}
runner_name: ${{ steps.ecs.outputs.runner_name }}
label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
steps:
- uses: actions/checkout@v4
with:
path: greptimedb
persist-credentials: false
- name: Create ephemeral ECS runner
if: inputs.provider == 'Aliyun'
id: ecs
uses: ./greptimedb/.github/actions/aliyun-ecs-create
with:
@@ -179,13 +189,35 @@ jobs:
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
instance-type: ${{ env.ECS_INSTANCE_TYPE }}
instance-type: ${{ needs.validate.outputs.instance_type }}
enable-docker: 'true'
system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
ttl-hours: ${{ inputs.janitor_ttl_hours }}
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
- uses: aws-actions/configure-aws-credentials@v4
if: inputs.provider == 'AWS'
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.EC2_RUNNER_REGION }}
- uses: astral-sh/setup-uv@v6
if: inputs.provider == 'AWS'
- name: Create ephemeral EC2 runner
if: inputs.provider == 'AWS'
id: aws
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
benchmark:
needs:
- validate
@@ -193,6 +225,7 @@ jobs:
runs-on: ${{ needs.provision.outputs.label }}
timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
env:
INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
TARGETS: ${{ needs.validate.outputs.targets }}
SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }}
@@ -209,6 +242,8 @@ jobs:
with:
path: greptimedb
persist-credentials: false
- name: Set up benchmark dependencies
uses: ./greptimedb/.github/actions/setup-benchmark
- name: Check Docker on ephemeral Linux host
shell: bash
run: |
@@ -252,9 +287,9 @@ jobs:
done
jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
--arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
--arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
--arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
--argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \
'{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
'{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ($provider | if . == "AWS" then "ec2" else "ecs" end):{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
- name: Checkout benchmark tools
uses: actions/checkout@v4
with:
@@ -343,7 +378,7 @@ jobs:
needs:
- provision
- benchmark
if: ${{ always() && needs.provision.outputs.instance_id != '' }}
if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
@@ -352,6 +387,7 @@ jobs:
path: greptimedb
persist-credentials: false
- name: Delete ECS and unregister runner
if: inputs.provider == 'Aliyun'
uses: ./greptimedb/.github/actions/aliyun-ecs-delete
with:
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
@@ -360,3 +396,20 @@ jobs:
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
instance-id: ${{ needs.provision.outputs.instance_id }}
runner-name: ${{ needs.provision.outputs.runner_name }}
- uses: aws-actions/configure-aws-credentials@v4
if: inputs.provider == 'AWS'
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.EC2_RUNNER_REGION }}
- uses: astral-sh/setup-uv@v6
if: inputs.provider == 'AWS'
- name: Delete EC2 and unregister runner
if: inputs.provider == 'AWS'
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py
.github/workflows/aws-ec2-runner-janitor.yml
+29
View File
@@ -0,0 +1,29 @@
name: AWS EC2 Runner Janitor
on:
schedule:
- cron: '23 3 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
sweep:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.EC2_RUNNER_REGION }}
- uses: astral-sh/setup-uv@v6
- name: Sweep expired CI instances and unregister runners
env:
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
run: uv run .github/scripts/aws-ec2-runner-teardown.py --sweep
.github/workflows/checks.yml
+1
View File
@@ -97,6 +97,7 @@ jobs:
python3 tests/perf/test_query_regression_nightly_refs.py
python3 tests/perf/test_query_regression_slash.py
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
python3 tests/perf/test_ci_runner_scripts.py
python3 tests/perf/test_agent_observability_summary.py
check:
.github/workflows/vmbench-long-range.yml
+73 -14
View File
@@ -35,10 +35,16 @@ on:
type: string
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
required: true
ecs_instance_type:
description: ECS instance type (independent of Query Regression)
provider:
description: Cloud provider for the benchmark runner
type: choice
options: [Aliyun, AWS]
default: Aliyun
required: true
instance_type:
description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
type: string
default: ecs.c9i.2xlarge
default: auto
required: true
benchmark_root:
description: Absolute disk-backed work directory (outside the o11ybench checkout)
@@ -46,9 +52,9 @@ on:
default: /home/runner/benchmark-data
required: true
system_disk_gib:
description: System disk GiB (20..2048)
description: System disk GiB (20..2048, default 500)
type: number
default: 80
default: 500
required: true
benchmark_timeout_minutes:
description: Benchmark job timeout in minutes (1..360)
@@ -78,7 +84,8 @@ env:
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
RUNTIME_IMAGE: ${{ inputs.runtime_image }}
ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
@@ -89,6 +96,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
instance_type: ${{ steps.runner.outputs.instance_type }}
system_disk_gib: ${{ steps.plan.outputs.system_disk_gib }}
runtime_image: ${{ steps.plan.outputs.runtime_image }}
o11ybench_ref: ${{ steps.plan.outputs.o11ybench_ref }}
@@ -99,8 +107,13 @@ jobs:
with:
path: greptimedb
persist-credentials: false
- name: Test existing ECS lifecycle helpers
run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
- name: Resolve CI runner
id: runner
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
- name: Test CI runner helpers
run: |
python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
python3 greptimedb/tests/perf/test_ci_runner_scripts.py
- name: Validate budgets and build warm plan
id: plan
env:
@@ -108,7 +121,6 @@ jobs:
RUN_VICTORIAMETRICS: ${{ inputs.run_victoriametrics }}
run: |
set -euo pipefail
[[ -n "$ECS_INSTANCE_TYPE" ]]
[[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]]
[[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* ]]
[[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]]
@@ -159,15 +171,16 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
outputs:
label: ${{ steps.ecs.outputs.label }}
instance_id: ${{ steps.ecs.outputs.instance_id }}
runner_name: ${{ steps.ecs.outputs.runner_name }}
label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
steps:
- uses: actions/checkout@v4
with:
path: greptimedb
persist-credentials: false
- name: Create ephemeral ECS runner
if: inputs.provider == 'Aliyun'
id: ecs
uses: ./greptimedb/.github/actions/aliyun-ecs-create
with:
@@ -178,13 +191,35 @@ jobs:
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
instance-type: ${{ env.ECS_INSTANCE_TYPE }}
instance-type: ${{ needs.validate.outputs.instance_type }}
enable-docker: 'true'
system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
ttl-hours: ${{ inputs.janitor_ttl_hours }}
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
- uses: aws-actions/configure-aws-credentials@v4
if: inputs.provider == 'AWS'
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.EC2_RUNNER_REGION }}
- uses: astral-sh/setup-uv@v6
if: inputs.provider == 'AWS'
- name: Create ephemeral EC2 runner
if: inputs.provider == 'AWS'
id: aws
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
benchmark:
needs:
- validate
@@ -202,6 +237,12 @@ jobs:
TSDG_BATCH_SIZE: '100000'
O11YBENCH_GENERATOR_CPUS: '4'
steps:
- uses: actions/checkout@v4
with:
path: greptimedb
persist-credentials: false
- name: Set up benchmark dependencies
uses: ./greptimedb/.github/actions/setup-benchmark
- name: Checkout matching benchmark tools
uses: actions/checkout@v4
with:
@@ -283,7 +324,7 @@ jobs:
needs:
- provision
- benchmark
if: ${{ always() && needs.provision.outputs.instance_id != '' }}
if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
@@ -292,6 +333,7 @@ jobs:
path: greptimedb
persist-credentials: false
- name: Delete ECS and unregister runner
if: inputs.provider == 'Aliyun'
uses: ./greptimedb/.github/actions/aliyun-ecs-delete
with:
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
@@ -300,3 +342,20 @@ jobs:
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
instance-id: ${{ needs.provision.outputs.instance_id }}
runner-name: ${{ needs.provision.outputs.runner_name }}
- uses: aws-actions/configure-aws-credentials@v4
if: inputs.provider == 'AWS'
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.EC2_RUNNER_REGION }}
- uses: astral-sh/setup-uv@v6
if: inputs.provider == 'AWS'
- name: Delete EC2 and unregister runner
if: inputs.provider == 'AWS'
env:
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py