mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-09-24 14:15:49 +00:00
ci: add manual agent observability benchmarks on Aliyun ECS (#9179)
* ci: add manual agent observability benchmarks on Aliyun ECS Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: configure observability ECS budgets and reuse runner actions Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: default observability runners to ecs.c9i.2xlarge Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(ci): prepare observability Docker access during ECS bootstrap Signed-off-by: WenyXu <wenymedia@gmail.com> * docs: remove standalone observability CI guide Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
@@ -0,0 +1,30 @@
|
||||
# Aliyun ECS runners
|
||||
|
||||
Use `aliyun-ecs-create` to create an ECS runner and `aliyun-ecs-delete` to
|
||||
remove the instance and unregister the runner. Both actions run on GitHub-hosted
|
||||
Linux runners after checkout. Pass credentials through action inputs.
|
||||
|
||||
Split the workflow into three jobs:
|
||||
|
||||
1. Create the runner from a prepared ECS image.
|
||||
2. Run the workload with `runs-on: ${{ needs.create.outputs.label }}`.
|
||||
3. Delete the runner in a job with `if: always()` and dependencies on both jobs.
|
||||
Pass `instance_id` and `runner_name` from the create job's outputs, even if
|
||||
provisioning fails after creating the instance.
|
||||
|
||||
See [agent-observability.yml](../../workflows/agent-observability.yml) for a caller
|
||||
and [action.yml](action.yml) for all inputs and outputs.
|
||||
|
||||
`system-disk-gib` defaults to 80. Optional `ttl-hours` sets when the janitor may
|
||||
remove the instance, measured from ECS creation. Without it, the janitor uses
|
||||
its four-hour fallback. Allow time for provisioning, queueing, the workload and
|
||||
teardown. Normal teardown does not wait for the TTL.
|
||||
|
||||
Deploy the TTL-aware janitor to the default branch before using a TTL above
|
||||
four hours; the old janitor ignores this setting. The actions retain the existing
|
||||
query-regression ownership tags and runner naming.
|
||||
|
||||
Set `enable-docker: 'true'` for container workloads. Root cloud-init starts the
|
||||
image-provided Docker CE service and adds the runner to the Docker group before
|
||||
starting the runner. Docker group access is root-equivalent. This option defaults
|
||||
to `false`, so existing query-regression jobs keep their current permissions.
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Create Aliyun ECS runner
|
||||
description: Reuse the ephemeral runner lifecycle, including existing janitor ownership
|
||||
tags.
|
||||
inputs:
|
||||
access-key-id:
|
||||
description: Aliyun access key ID
|
||||
required: true
|
||||
access-key-secret:
|
||||
description: Aliyun access key secret
|
||||
required: true
|
||||
github-token:
|
||||
description: Token authorized to register and remove repository runners
|
||||
required: true
|
||||
region-id:
|
||||
description: ECS region
|
||||
required: true
|
||||
vswitch-id:
|
||||
description: ECS VSwitch
|
||||
required: true
|
||||
security-group-id:
|
||||
description: ECS security group
|
||||
required: true
|
||||
image-id:
|
||||
description: Prepared ephemeral runner ECS image
|
||||
required: true
|
||||
instance-type:
|
||||
description: ECS instance type
|
||||
required: true
|
||||
system-disk-gib:
|
||||
description: System disk size in GiB (20..2048)
|
||||
required: false
|
||||
default: '80'
|
||||
ttl-hours:
|
||||
description: Optional per-instance janitor TTL (1..168 hours); empty preserves the janitor fallback
|
||||
required: false
|
||||
default: ''
|
||||
enable-docker:
|
||||
description: Start image-provided Docker and grant the runner Docker group access (root-equivalent)
|
||||
required: false
|
||||
default: 'false'
|
||||
resource-group-id:
|
||||
description: Optional ECS resource group
|
||||
required: false
|
||||
default: ''
|
||||
runner-uid:
|
||||
description: Runner user ID baked into the image
|
||||
required: false
|
||||
default: '1001'
|
||||
runner-gid:
|
||||
description: Runner group ID baked into the image
|
||||
required: false
|
||||
default: '1001'
|
||||
outputs:
|
||||
label:
|
||||
description: label
|
||||
value: ${{ steps.provision.outputs.label }}
|
||||
instance_id:
|
||||
description: instance_id
|
||||
value: ${{ steps.provision.outputs.instance_id }}
|
||||
runner_name:
|
||||
description: runner_name
|
||||
value: ${{ steps.provision.outputs.runner_name }}
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- id: provision
|
||||
shell: bash
|
||||
env:
|
||||
ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
|
||||
ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }}
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }}
|
||||
ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }}
|
||||
ALIYUN_ECS_VSWITCH_ID: ${{ inputs.vswitch-id }}
|
||||
ALIYUN_ECS_SECURITY_GROUP_ID: ${{ inputs.security-group-id }}
|
||||
QUERY_REGRESSION_ECS_IMAGE_ID: ${{ inputs.image-id }}
|
||||
ALIYUN_ECS_INSTANCE_TYPE: ${{ inputs.instance-type }}
|
||||
ALIYUN_ECS_SYSTEM_DISK_GIB: ${{ inputs.system-disk-gib }}
|
||||
ALIYUN_ECS_TTL_HOURS: ${{ inputs.ttl-hours }}
|
||||
ALIYUN_ECS_ENABLE_DOCKER: ${{ inputs.enable-docker }}
|
||||
ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ inputs.resource-group-id }}
|
||||
QUERY_REGRESSION_RUNNER_UID: ${{ inputs.runner-uid }}
|
||||
QUERY_REGRESSION_RUNNER_GID: ${{ inputs.runner-gid }}
|
||||
ACTION_PATH: ${{ github.action_path }}
|
||||
run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-provision.py"
|
||||
@@ -0,0 +1,37 @@
|
||||
name: Delete Aliyun ECS runner
|
||||
description: Reuse the ephemeral runner lifecycle, including existing janitor ownership
|
||||
tags.
|
||||
inputs:
|
||||
access-key-id:
|
||||
description: Aliyun access key ID
|
||||
required: true
|
||||
access-key-secret:
|
||||
description: Aliyun access key secret
|
||||
required: true
|
||||
github-token:
|
||||
description: Token authorized to register and remove repository runners
|
||||
required: true
|
||||
region-id:
|
||||
description: ECS region
|
||||
required: true
|
||||
instance-id:
|
||||
description: Instance ID returned by create
|
||||
required: true
|
||||
runner-name:
|
||||
description: Runner name returned by create
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- id: teardown
|
||||
shell: bash
|
||||
env:
|
||||
ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
|
||||
ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }}
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }}
|
||||
ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }}
|
||||
QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ inputs.instance-id }}
|
||||
QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ inputs.runner-name }}
|
||||
ACTION_PATH: ${{ github.action_path }}
|
||||
run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-teardown.py"
|
||||
@@ -58,6 +58,7 @@ RUNNER_LABEL_PREFIX = "query-regression-ecs"
|
||||
MANAGED_BY_TAG_KEY = "managed-by"
|
||||
MANAGED_BY_TAG_VALUE = "query-regression-ci"
|
||||
RUN_TAG_KEY = "query-regression-run-id"
|
||||
TTL_TAG_KEY = "runner-ttl-hours"
|
||||
|
||||
# Runner cache paths on the instance system disk. They are created empty
|
||||
# every provision and deleted with the VM.
|
||||
@@ -100,6 +101,15 @@ class ProvisionConfig:
|
||||
# Runner identity inside the image; the workflow asserts the same values.
|
||||
runner_uid: str = "1001"
|
||||
runner_gid: str = "1001"
|
||||
system_disk_gib: int = SYSTEM_DISK_GIB
|
||||
ttl_hours: int | None = None
|
||||
enable_docker: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 20 <= self.system_disk_gib <= 2048:
|
||||
raise ValueError("system-disk-gib must be between 20 and 2048")
|
||||
if self.ttl_hours is not None and not 1 <= self.ttl_hours <= 168:
|
||||
raise ValueError("ttl-hours must be between 1 and 168")
|
||||
|
||||
|
||||
def runner_name_for_run(run_id: str) -> str:
|
||||
@@ -117,8 +127,20 @@ def render_user_data(
|
||||
repo: str,
|
||||
runner_uid: str = "1001",
|
||||
runner_gid: str = "1001",
|
||||
enable_docker: bool = False,
|
||||
) -> str:
|
||||
"""Render the cloud-init shell script for the runner instance."""
|
||||
docker_setup = ""
|
||||
if enable_docker:
|
||||
docker_setup = f'''# Reuse Docker CE from the ECS image; run before dropping runner privileges.
|
||||
command -v docker
|
||||
command -v jq
|
||||
systemctl start docker
|
||||
runner_user=$(id -nu {runner_uid})
|
||||
usermod -aG docker "$runner_user"
|
||||
runuser -u "$runner_user" -- docker info
|
||||
|
||||
'''
|
||||
destinations = " ".join(f'"{dst}"' for dst in CACHE_PATHS)
|
||||
cache_setup = f"""# Caches live on the system disk, are deleted with the instance, and every
|
||||
# run compiles cold. Within-run reuse (base warming candidate via the shared
|
||||
@@ -158,7 +180,7 @@ set -euo pipefail
|
||||
|
||||
{swap_setup}
|
||||
|
||||
cat > /etc/ephemeral-github-runner.env <<'ENVEOF'
|
||||
{docker_setup}cat > /etc/ephemeral-github-runner.env <<'ENVEOF'
|
||||
RUNNER_NAME={runner_name}
|
||||
RUNNER_LABELS={runner_label}
|
||||
RUNNER_TOKEN={runner_token}
|
||||
@@ -372,13 +394,24 @@ def run_instance(client, config: ProvisionConfig, user_data: str) -> str:
|
||||
internet_max_bandwidth_out=100,
|
||||
system_disk=ecs_models.RunInstancesRequestSystemDisk(
|
||||
category="cloud_essd",
|
||||
size=str(SYSTEM_DISK_GIB),
|
||||
size=str(config.system_disk_gib),
|
||||
),
|
||||
user_data=user_data,
|
||||
tag=[
|
||||
ecs_models.RunInstancesRequestTag(key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE),
|
||||
ecs_models.RunInstancesRequestTag(
|
||||
key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE
|
||||
),
|
||||
ecs_models.RunInstancesRequestTag(key=RUN_TAG_KEY, value=config.run_id),
|
||||
],
|
||||
]
|
||||
+ (
|
||||
[
|
||||
ecs_models.RunInstancesRequestTag(
|
||||
key=TTL_TAG_KEY, value=str(config.ttl_hours)
|
||||
)
|
||||
]
|
||||
if config.ttl_hours is not None
|
||||
else []
|
||||
),
|
||||
)
|
||||
response = client.run_instances(request)
|
||||
instance_id = response.body.instance_id_sets.instance_id_set[0]
|
||||
@@ -399,6 +432,7 @@ def provision(config: ProvisionConfig) -> int:
|
||||
config.repo,
|
||||
config.runner_uid,
|
||||
config.runner_gid,
|
||||
enable_docker=config.enable_docker,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -450,6 +484,18 @@ def main() -> int:
|
||||
parser.add_argument("--security-group-id", default=os.environ.get("ALIYUN_ECS_SECURITY_GROUP_ID"))
|
||||
parser.add_argument("--image-id", default=os.environ.get("QUERY_REGRESSION_ECS_IMAGE_ID"))
|
||||
parser.add_argument("--instance-type", default=os.environ.get("ALIYUN_ECS_INSTANCE_TYPE"))
|
||||
parser.add_argument(
|
||||
"--system-disk-gib",
|
||||
type=int,
|
||||
default=os.environ.get("ALIYUN_ECS_SYSTEM_DISK_GIB", str(SYSTEM_DISK_GIB)),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ttl-hours", type=int, default=os.environ.get("ALIYUN_ECS_TTL_HOURS") or None
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-docker", choices=("true", "false"),
|
||||
default=os.environ.get("ALIYUN_ECS_ENABLE_DOCKER", "false"),
|
||||
)
|
||||
parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY"))
|
||||
parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID"))
|
||||
parser.add_argument("--github-token", default=os.environ.get("GH_PERSONAL_ACCESS_TOKEN"))
|
||||
@@ -461,7 +507,7 @@ def main() -> int:
|
||||
missing = [
|
||||
name
|
||||
for name, value in vars(args).items()
|
||||
if name not in ("resource_group_id",)
|
||||
if name not in ("resource_group_id", "ttl_hours")
|
||||
and (value is None or (isinstance(value, str) and not value))
|
||||
]
|
||||
if missing:
|
||||
@@ -480,6 +526,9 @@ def main() -> int:
|
||||
resource_group_id=args.resource_group_id or None,
|
||||
runner_uid=args.runner_uid,
|
||||
runner_gid=args.runner_gid,
|
||||
system_disk_gib=args.system_disk_gib,
|
||||
ttl_hours=args.ttl_hours,
|
||||
enable_docker=args.enable_docker == "true",
|
||||
)
|
||||
return provision(config)
|
||||
|
||||
|
||||
@@ -32,7 +32,8 @@ Two modes:
|
||||
runner by name. Both steps are idempotent and best-effort; a missing
|
||||
instance or runner is not an error.
|
||||
- Sweep (scheduled janitor): delete every instance tagged as managed by
|
||||
query-regression CI whose creation time is older than the given TTL, and
|
||||
query-regression CI whose creation time is older than its tagged TTL (or
|
||||
the given fallback TTL for untagged instances), and
|
||||
deregister the matching runners. This is the safety net for runs whose
|
||||
teardown job never executed.
|
||||
"""
|
||||
@@ -72,16 +73,31 @@ def parse_creation_time(value: str) -> datetime:
|
||||
|
||||
|
||||
def expired_instance_names(
|
||||
instances: list[tuple[str, str, str]], now: datetime, ttl: timedelta
|
||||
instances: list[tuple[str, str, str, str | None]], now: datetime, ttl: timedelta
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Pick (instance_id, instance_name) pairs whose creation is older than ttl.
|
||||
|
||||
`instances` items are (instance_id, instance_name, creation_time).
|
||||
`instances` items are (instance_id, instance_name, creation_time, tagged_ttl_hours).
|
||||
"""
|
||||
expired = []
|
||||
for instance_id, instance_name, creation_time in instances:
|
||||
for instance_id, instance_name, creation_time, tagged_ttl in instances:
|
||||
instance_ttl = ttl
|
||||
if tagged_ttl is not None:
|
||||
try:
|
||||
hours = int(tagged_ttl)
|
||||
if not 1 <= hours <= 168:
|
||||
raise ValueError("TTL outside 1..168 hours")
|
||||
instance_ttl = timedelta(hours=hours)
|
||||
except (TypeError, ValueError):
|
||||
# Never fall back to a shorter TTL and kill a live tagged runner.
|
||||
print(
|
||||
f"Skipping {instance_id}: invalid {provision.TTL_TAG_KEY}={tagged_ttl!r}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
age = now - parse_creation_time(creation_time)
|
||||
if age >= ttl:
|
||||
if age >= instance_ttl:
|
||||
expired.append((instance_id, instance_name))
|
||||
return expired
|
||||
|
||||
@@ -157,17 +173,20 @@ def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]]:
|
||||
def list_managed_instances(
|
||||
client, region_id: str
|
||||
) -> list[tuple[str, str, str, str | None]]:
|
||||
from alibabacloud_ecs20140526 import models as ecs_models
|
||||
|
||||
result: list[tuple[str, str, str]] = []
|
||||
result: list[tuple[str, str, str, str | None]] = []
|
||||
next_token = None
|
||||
while True:
|
||||
request = ecs_models.DescribeInstancesRequest(
|
||||
region_id=region_id,
|
||||
tag=[
|
||||
ecs_models.DescribeInstancesRequestTag(
|
||||
key=provision.MANAGED_BY_TAG_KEY, value=provision.MANAGED_BY_TAG_VALUE
|
||||
key=provision.MANAGED_BY_TAG_KEY,
|
||||
value=provision.MANAGED_BY_TAG_VALUE,
|
||||
)
|
||||
],
|
||||
max_results=100,
|
||||
@@ -175,7 +194,19 @@ def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]]
|
||||
)
|
||||
response = client.describe_instances(request)
|
||||
for instance in response.body.instances.instance:
|
||||
result.append((instance.instance_id, instance.instance_name, instance.creation_time))
|
||||
tags = (instance.tags.tag or []) if instance.tags else []
|
||||
tagged_ttl = next(
|
||||
(tag.tag_value for tag in tags if tag.tag_key == provision.TTL_TAG_KEY),
|
||||
None,
|
||||
)
|
||||
result.append(
|
||||
(
|
||||
instance.instance_id,
|
||||
instance.instance_name,
|
||||
instance.creation_time,
|
||||
tagged_ttl,
|
||||
)
|
||||
)
|
||||
next_token = response.body.next_token
|
||||
if not next_token:
|
||||
return result
|
||||
@@ -187,7 +218,10 @@ def sweep(client, region_id: str, repo: str, github_token: str, ttl: timedelta)
|
||||
expired = expired_instance_names(instances, datetime.now(timezone.utc), ttl)
|
||||
ok = True
|
||||
for instance_id, instance_name in expired:
|
||||
print(f"Instance {instance_id} ({instance_name}) exceeds TTL {ttl}; deleting", flush=True)
|
||||
print(
|
||||
f"Instance {instance_id} ({instance_name}) exceeds its configured TTL; deleting",
|
||||
flush=True,
|
||||
)
|
||||
# Runner names mirror instance names by construction in the provision
|
||||
# script (both are qreg-ecs-<run_id>).
|
||||
ok &= delete_instance(client, instance_id, region_id)
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
name: Agent Observability Benchmark
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
targets:
|
||||
description: all or comma-separated greptimedb,clickhouse,victorialogs
|
||||
type: string
|
||||
default: greptimedb
|
||||
required: true
|
||||
dataset:
|
||||
description: 'Dataset size: S=5K, P=10M, M=100M'
|
||||
type: choice
|
||||
options:
|
||||
- S
|
||||
- P
|
||||
- M
|
||||
default: S
|
||||
required: true
|
||||
greptimedb_tag:
|
||||
description: Docker Hub greptime/greptimedb tag
|
||||
type: string
|
||||
default: latest
|
||||
required: true
|
||||
clickhouse_tag:
|
||||
description: Docker Hub clickhouse/clickhouse-server tag
|
||||
type: string
|
||||
default: 26.6.1.1193
|
||||
required: true
|
||||
victorialogs_tag:
|
||||
description: Docker Hub victoriametrics/victoria-logs tag or tag@sha256 digest
|
||||
type: string
|
||||
default: v1.52.0@sha256:47b820890d64c4575a2a0a46415dcd8a4fd59a0f1fcd6a377693d7aea639442e
|
||||
required: true
|
||||
o11ybench_ref:
|
||||
description: Optional full commit SHA override (default uses runtime image revision)
|
||||
type: string
|
||||
required: false
|
||||
runtime_image:
|
||||
description: Optional Aliyun runtime image override (default uses repository variable)
|
||||
type: string
|
||||
required: false
|
||||
ecs_instance_type:
|
||||
description: ECS instance type (independent of Query Regression)
|
||||
type: string
|
||||
default: ecs.c9i.2xlarge
|
||||
required: true
|
||||
system_disk_gib:
|
||||
description: ECS system disk size in GiB (20..2048; includes corpus, DB, images and swap)
|
||||
type: number
|
||||
default: 500
|
||||
required: true
|
||||
benchmark_timeout_minutes:
|
||||
description: Benchmark job timeout in minutes (1..360)
|
||||
type: number
|
||||
default: 360
|
||||
required: true
|
||||
janitor_ttl_hours:
|
||||
description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h)
|
||||
type: number
|
||||
default: 8
|
||||
required: true
|
||||
db_cpus:
|
||||
description: CPU limit per DB container
|
||||
type: string
|
||||
default: '4'
|
||||
required: true
|
||||
db_memory:
|
||||
description: Memory limit per DB container
|
||||
type: string
|
||||
default: 8g
|
||||
required: true
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
TARGETS: ${{ inputs.targets }}
|
||||
PROFILE: ${{ inputs.dataset }}
|
||||
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
|
||||
CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
|
||||
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
|
||||
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image || vars.O11YBENCH_RUNTIME_IMAGE }}
|
||||
ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
|
||||
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
|
||||
DB_CPUS: ${{ inputs.db_cpus }}
|
||||
DB_MEMORY: ${{ inputs.db_memory }}
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
greptimedb: ${{ steps.inputs.outputs.greptimedb }}
|
||||
clickhouse: ${{ steps.inputs.outputs.clickhouse }}
|
||||
victorialogs: ${{ steps.inputs.outputs.victorialogs }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Test ECS lifecycle helpers
|
||||
run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
- name: Validate requested benchmark
|
||||
id: inputs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Validate cloud lifecycle budgets before provisioning; runtime owns workload validation.
|
||||
[[ -n "$ECS_INSTANCE_TYPE" ]]
|
||||
for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
|
||||
[[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; }
|
||||
done
|
||||
((SYSTEM_DISK_GIB >= 20 && SYSTEM_DISK_GIB <= 2048))
|
||||
((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
|
||||
# Reserve 45m provision + 15m teardown + 60m queue headroom before expiry.
|
||||
((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) || {
|
||||
echo 'Janitor TTL must cover benchmark timeout plus 2 hours' >&2; exit 1;
|
||||
}
|
||||
[[ -z "$O11YBENCH_REF" || "$O11YBENCH_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ -n "$RUNTIME_IMAGE" ]] || { echo 'Configure O11YBENCH_RUNTIME_IMAGE or supply runtime_image' >&2; exit 1; }
|
||||
selected="$TARGETS"
|
||||
[[ "$selected" != all ]] || selected=greptimedb,clickhouse,victorialogs
|
||||
[[ "$selected" =~ ^(greptimedb|clickhouse|victorialogs)(,(greptimedb|clickhouse|victorialogs))*$ ]]
|
||||
declare -A enabled
|
||||
IFS=, read -ra targets <<< "$selected"
|
||||
for target in "${targets[@]}"; do
|
||||
[[ -z ${enabled[$target]:-} ]] || { echo "Duplicate target: $target" >&2; exit 1; }
|
||||
enabled[$target]=true
|
||||
done
|
||||
for target in greptimedb clickhouse victorialogs; do
|
||||
printf '%s=%s\n' "$target" "${enabled[$target]:-false}" >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
provision:
|
||||
needs: validate
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
label: ${{ steps.ecs.outputs.label }}
|
||||
instance_id: ${{ steps.ecs.outputs.instance_id }}
|
||||
runner_name: ${{ steps.ecs.outputs.runner_name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Create ephemeral ECS runner
|
||||
id: ecs
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-create
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
|
||||
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
|
||||
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
|
||||
instance-type: ${{ env.ECS_INSTANCE_TYPE }}
|
||||
enable-docker: 'true'
|
||||
system-disk-gib: ${{ inputs.system_disk_gib }}
|
||||
ttl-hours: ${{ inputs.janitor_ttl_hours }}
|
||||
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
|
||||
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
|
||||
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
|
||||
benchmark:
|
||||
needs:
|
||||
- validate
|
||||
- provision
|
||||
runs-on: ${{ needs.provision.outputs.label }}
|
||||
timeout-minutes: ${{ inputs.benchmark_timeout_minutes }}
|
||||
env:
|
||||
OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
ARTIFACT_ROOT: ${{ github.workspace }}/benchmark-data/${{ github.run_id }}-${{ github.run_attempt }}
|
||||
MANIFEST_PATH: ${{ github.workspace }}/run-manifest.json
|
||||
O11YBENCH_DIR: ${{ github.workspace }}/o11ybench
|
||||
GREPTIMEDB_HTTP_PORT: '14000'
|
||||
GREPTIMEDB_GRPC_PORT: '14001'
|
||||
CLICKHOUSE_HTTP_PORT: '18123'
|
||||
CLICKHOUSE_NATIVE_PORT: '19000'
|
||||
VICTORIALOGS_HTTP_PORT: '19428'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Check Docker on ephemeral Linux host
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$(uname -m)" == x86_64 ]]
|
||||
jq --version
|
||||
docker info
|
||||
# The workspace (and Docker state) must not use a small /tmp filesystem.
|
||||
df -h "$GITHUB_WORKSPACE" /var/lib/docker
|
||||
- name: Resolve image tags and verify runtime revision
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
actual=
|
||||
[[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
|
||||
images='{}'
|
||||
resolve_image() {
|
||||
local target=$1 reference=$2 variable=$3 metadata id revision
|
||||
docker pull "$reference"
|
||||
metadata=$(docker image inspect "$reference")
|
||||
id=$(jq -er '.[0].Id' <<< "$metadata")
|
||||
if [[ "$target" == runtime ]]; then
|
||||
revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
|
||||
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || { echo 'Runtime lacks a full commit revision' >&2; return 1; }
|
||||
[[ -z "$O11YBENCH_REF" || "$revision" == "$O11YBENCH_REF" ]] || { echo 'Runtime and requested o11ybench revisions differ' >&2; return 1; }
|
||||
actual=$revision
|
||||
printf 'RESOLVED_O11YBENCH_REF=%s\n' "$revision" >> "$GITHUB_ENV"
|
||||
fi
|
||||
printf '%s=%s\n' "$variable" "$id" >> "$GITHUB_ENV"
|
||||
images=$(jq --arg target "$target" --arg requested "$reference" --argjson metadata "$metadata" \
|
||||
'. + {($target): {requested: $requested, id: $metadata[0].Id, digests: ($metadata[0].RepoDigests // [])}}' <<< "$images")
|
||||
}
|
||||
resolve_image runtime "$RUNTIME_IMAGE" RESOLVED_RUNTIME_IMAGE
|
||||
for target in greptimedb clickhouse victorialogs; do
|
||||
[[ "$TARGETS" == all || ",$TARGETS," == *",$target,"* ]] || continue
|
||||
case "$target" in
|
||||
greptimedb) resolve_image "$target" "greptime/greptimedb:$GREPTIMEDB_TAG" GREPTIMEDB_IMAGE ;;
|
||||
clickhouse) resolve_image "$target" "clickhouse/clickhouse-server:$CLICKHOUSE_TAG" CLICKHOUSE_IMAGE ;;
|
||||
victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
|
||||
esac
|
||||
done
|
||||
jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
|
||||
--arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
|
||||
--arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
|
||||
--argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \
|
||||
'{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
|
||||
- name: Checkout benchmark tools
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: GreptimeTeam/o11ybench
|
||||
ref: ${{ env.RESOLVED_O11YBENCH_REF }}
|
||||
path: o11ybench
|
||||
persist-credentials: false
|
||||
- name: Verify benchmark checkout revision
|
||||
shell: bash
|
||||
run: test "$(git -C "$O11YBENCH_DIR" rev-parse HEAD)" = "$RESOLVED_O11YBENCH_REF"
|
||||
- name: Generate dataset once
|
||||
id: generate
|
||||
run: |-
|
||||
bash o11ybench/scripts/run-agent-observability.sh --stage generate --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
|
||||
cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
|
||||
- name: Load and benchmark greptimedb
|
||||
id: greptimedb
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
|
||||
run: |-
|
||||
bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark clickhouse
|
||||
id: clickhouse
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }}
|
||||
run: |-
|
||||
bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target clickhouse --image "$CLICKHOUSE_IMAGE" \
|
||||
--clickhouse-http-port "$CLICKHOUSE_HTTP_PORT" --clickhouse-native-port "$CLICKHOUSE_NATIVE_PORT"
|
||||
- name: Load and benchmark victorialogs
|
||||
id: victorialogs
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victorialogs == 'true' }}
|
||||
run: |-
|
||||
bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target victorialogs --image "$VICTORIALOGS_IMAGE" \
|
||||
--victorialogs-http-port "$VICTORIALOGS_HTTP_PORT"
|
||||
- name: Compare all three targets
|
||||
if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
|
||||
== 'success' }}
|
||||
run: |-
|
||||
docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
|
||||
--mount "type=bind,src=$ARTIFACT_ROOT,dst=/results" "$RESOLVED_RUNTIME_IMAGE" \
|
||||
scripts/finalize_logbench_agent_observability_measured.py \
|
||||
--greptimedb-summary /results/greptimedb-1/greptimedb-1/measured-summary.json \
|
||||
--clickhouse-summary /results/clickhouse-1/clickhouse-1/measured-summary.json \
|
||||
--victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \
|
||||
--output /results/comparison.json
|
||||
- name: Clean up owned containers
|
||||
if: always()
|
||||
run: |-
|
||||
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
|
||||
bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
|
||||
--owner "$OWNER" --root "$ARTIFACT_ROOT"
|
||||
fi
|
||||
- name: Upload results (no dataset)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |-
|
||||
${{ env.MANIFEST_PATH }}
|
||||
${{ env.ARTIFACT_ROOT }}
|
||||
!${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
teardown:
|
||||
needs:
|
||||
- provision
|
||||
- benchmark
|
||||
if: ${{ always() && needs.provision.outputs.instance_id != '' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Delete ECS and unregister runner
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-delete
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
instance-id: ${{ needs.provision.outputs.instance_id }}
|
||||
runner-name: ${{ needs.provision.outputs.runner_name }}
|
||||
@@ -157,25 +157,24 @@ jobs:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
|
||||
- name: Provision ECS runner
|
||||
id: provision
|
||||
env:
|
||||
ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
ALIYUN_ECS_VSWITCH_ID: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
|
||||
ALIYUN_ECS_SECURITY_GROUP_ID: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
|
||||
ALIYUN_ECS_INSTANCE_TYPE: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }}
|
||||
ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
|
||||
QUERY_REGRESSION_ECS_IMAGE_ID: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
|
||||
QUERY_REGRESSION_RUNNER_UID: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
|
||||
QUERY_REGRESSION_RUNNER_GID: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
|
||||
run: >-
|
||||
uv run .github/scripts/aliyun-ecs-runner-provision.py
|
||||
uses: ./.github/actions/aliyun-ecs-create
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
|
||||
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
|
||||
instance-type: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }}
|
||||
# Preserve query regression's original resource and janitor budgets.
|
||||
system-disk-gib: '80'
|
||||
ttl-hours: '4'
|
||||
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
|
||||
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
|
||||
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
|
||||
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
|
||||
|
||||
query-regression:
|
||||
needs: [provision, test-tooling]
|
||||
@@ -896,16 +895,12 @@ jobs:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
|
||||
- name: Teardown ECS runner
|
||||
env:
|
||||
ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
|
||||
QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
|
||||
run: >-
|
||||
uv run .github/scripts/aliyun-ecs-runner-teardown.py
|
||||
uses: ./.github/actions/aliyun-ecs-delete
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
|
||||
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
instance-id: ${{ needs.provision.outputs.instance_id }}
|
||||
runner-name: ${{ needs.provision.outputs.runner_name }}
|
||||
|
||||
Reference in New Issue
Block a user