diff --git a/.github/actions/aliyun-ecs-create/README.md b/.github/actions/aliyun-ecs-create/README.md new file mode 100644 index 0000000000..eb712c0a7d --- /dev/null +++ b/.github/actions/aliyun-ecs-create/README.md @@ -0,0 +1,30 @@ +# Aliyun ECS runners + +Use `aliyun-ecs-create` to create an ECS runner and `aliyun-ecs-delete` to +remove the instance and unregister the runner. Both actions run on GitHub-hosted +Linux runners after checkout. Pass credentials through action inputs. + +Split the workflow into three jobs: + +1. Create the runner from a prepared ECS image. +2. Run the workload with `runs-on: ${{ needs.create.outputs.label }}`. +3. Delete the runner in a job with `if: always()` and dependencies on both jobs. + Pass `instance_id` and `runner_name` from the create job's outputs, even if + provisioning fails after creating the instance. + +See [agent-observability.yml](../../workflows/agent-observability.yml) for a caller +and [action.yml](action.yml) for all inputs and outputs. + +`system-disk-gib` defaults to 80. Optional `ttl-hours` sets when the janitor may +remove the instance, measured from ECS creation. Without it, the janitor uses +its four-hour fallback. Allow time for provisioning, queueing, the workload and +teardown. Normal teardown does not wait for the TTL. + +Deploy the TTL-aware janitor to the default branch before using a TTL above +four hours; the old janitor ignores this setting. The actions retain the existing +query-regression ownership tags and runner naming. + +Set `enable-docker: 'true'` for container workloads. Root cloud-init starts the +image-provided Docker CE service and adds the runner to the Docker group before +starting the runner. Docker group access is root-equivalent. This option defaults +to `false`, so existing query-regression jobs keep their current permissions. diff --git a/.github/actions/aliyun-ecs-create/action.yml b/.github/actions/aliyun-ecs-create/action.yml new file mode 100644 index 0000000000..213eb46863 --- /dev/null +++ b/.github/actions/aliyun-ecs-create/action.yml @@ -0,0 +1,85 @@ +name: Create Aliyun ECS runner +description: Reuse the ephemeral runner lifecycle, including existing janitor ownership + tags. +inputs: + access-key-id: + description: Aliyun access key ID + required: true + access-key-secret: + description: Aliyun access key secret + required: true + github-token: + description: Token authorized to register and remove repository runners + required: true + region-id: + description: ECS region + required: true + vswitch-id: + description: ECS VSwitch + required: true + security-group-id: + description: ECS security group + required: true + image-id: + description: Prepared ephemeral runner ECS image + required: true + instance-type: + description: ECS instance type + required: true + system-disk-gib: + description: System disk size in GiB (20..2048) + required: false + default: '80' + ttl-hours: + description: Optional per-instance janitor TTL (1..168 hours); empty preserves the janitor fallback + required: false + default: '' + enable-docker: + description: Start image-provided Docker and grant the runner Docker group access (root-equivalent) + required: false + default: 'false' + resource-group-id: + description: Optional ECS resource group + required: false + default: '' + runner-uid: + description: Runner user ID baked into the image + required: false + default: '1001' + runner-gid: + description: Runner group ID baked into the image + required: false + default: '1001' +outputs: + label: + description: label + value: ${{ steps.provision.outputs.label }} + instance_id: + description: instance_id + value: ${{ steps.provision.outputs.instance_id }} + runner_name: + description: runner_name + value: ${{ steps.provision.outputs.runner_name }} +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@v6 + - id: provision + shell: bash + env: + ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }} + ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }} + GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }} + ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }} + ALIYUN_ECS_VSWITCH_ID: ${{ inputs.vswitch-id }} + ALIYUN_ECS_SECURITY_GROUP_ID: ${{ inputs.security-group-id }} + QUERY_REGRESSION_ECS_IMAGE_ID: ${{ inputs.image-id }} + ALIYUN_ECS_INSTANCE_TYPE: ${{ inputs.instance-type }} + ALIYUN_ECS_SYSTEM_DISK_GIB: ${{ inputs.system-disk-gib }} + ALIYUN_ECS_TTL_HOURS: ${{ inputs.ttl-hours }} + ALIYUN_ECS_ENABLE_DOCKER: ${{ inputs.enable-docker }} + ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ inputs.resource-group-id }} + QUERY_REGRESSION_RUNNER_UID: ${{ inputs.runner-uid }} + QUERY_REGRESSION_RUNNER_GID: ${{ inputs.runner-gid }} + ACTION_PATH: ${{ github.action_path }} + run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-provision.py" diff --git a/.github/actions/aliyun-ecs-delete/action.yml b/.github/actions/aliyun-ecs-delete/action.yml new file mode 100644 index 0000000000..081b15395e --- /dev/null +++ b/.github/actions/aliyun-ecs-delete/action.yml @@ -0,0 +1,37 @@ +name: Delete Aliyun ECS runner +description: Reuse the ephemeral runner lifecycle, including existing janitor ownership + tags. +inputs: + access-key-id: + description: Aliyun access key ID + required: true + access-key-secret: + description: Aliyun access key secret + required: true + github-token: + description: Token authorized to register and remove repository runners + required: true + region-id: + description: ECS region + required: true + instance-id: + description: Instance ID returned by create + required: true + runner-name: + description: Runner name returned by create + required: true +runs: + using: composite + steps: + - uses: astral-sh/setup-uv@v6 + - id: teardown + shell: bash + env: + ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }} + ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }} + GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }} + ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }} + QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ inputs.instance-id }} + QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ inputs.runner-name }} + ACTION_PATH: ${{ github.action_path }} + run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-teardown.py" diff --git a/.github/scripts/aliyun-ecs-runner-provision.py b/.github/scripts/aliyun-ecs-runner-provision.py index b5dee7f51f..8c80e11cee 100644 --- a/.github/scripts/aliyun-ecs-runner-provision.py +++ b/.github/scripts/aliyun-ecs-runner-provision.py @@ -58,6 +58,7 @@ RUNNER_LABEL_PREFIX = "query-regression-ecs" MANAGED_BY_TAG_KEY = "managed-by" MANAGED_BY_TAG_VALUE = "query-regression-ci" RUN_TAG_KEY = "query-regression-run-id" +TTL_TAG_KEY = "runner-ttl-hours" # Runner cache paths on the instance system disk. They are created empty # every provision and deleted with the VM. @@ -100,6 +101,15 @@ class ProvisionConfig: # Runner identity inside the image; the workflow asserts the same values. runner_uid: str = "1001" runner_gid: str = "1001" + system_disk_gib: int = SYSTEM_DISK_GIB + ttl_hours: int | None = None + enable_docker: bool = False + + def __post_init__(self) -> None: + if not 20 <= self.system_disk_gib <= 2048: + raise ValueError("system-disk-gib must be between 20 and 2048") + if self.ttl_hours is not None and not 1 <= self.ttl_hours <= 168: + raise ValueError("ttl-hours must be between 1 and 168") def runner_name_for_run(run_id: str) -> str: @@ -117,8 +127,20 @@ def render_user_data( repo: str, runner_uid: str = "1001", runner_gid: str = "1001", + enable_docker: bool = False, ) -> str: """Render the cloud-init shell script for the runner instance.""" + docker_setup = "" + if enable_docker: + docker_setup = f'''# Reuse Docker CE from the ECS image; run before dropping runner privileges. +command -v docker +command -v jq +systemctl start docker +runner_user=$(id -nu {runner_uid}) +usermod -aG docker "$runner_user" +runuser -u "$runner_user" -- docker info + +''' destinations = " ".join(f'"{dst}"' for dst in CACHE_PATHS) cache_setup = f"""# Caches live on the system disk, are deleted with the instance, and every # run compiles cold. Within-run reuse (base warming candidate via the shared @@ -158,7 +180,7 @@ set -euo pipefail {swap_setup} -cat > /etc/ephemeral-github-runner.env <<'ENVEOF' +{docker_setup}cat > /etc/ephemeral-github-runner.env <<'ENVEOF' RUNNER_NAME={runner_name} RUNNER_LABELS={runner_label} RUNNER_TOKEN={runner_token} @@ -372,13 +394,24 @@ def run_instance(client, config: ProvisionConfig, user_data: str) -> str: internet_max_bandwidth_out=100, system_disk=ecs_models.RunInstancesRequestSystemDisk( category="cloud_essd", - size=str(SYSTEM_DISK_GIB), + size=str(config.system_disk_gib), ), user_data=user_data, tag=[ - ecs_models.RunInstancesRequestTag(key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE), + ecs_models.RunInstancesRequestTag( + key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE + ), ecs_models.RunInstancesRequestTag(key=RUN_TAG_KEY, value=config.run_id), - ], + ] + + ( + [ + ecs_models.RunInstancesRequestTag( + key=TTL_TAG_KEY, value=str(config.ttl_hours) + ) + ] + if config.ttl_hours is not None + else [] + ), ) response = client.run_instances(request) instance_id = response.body.instance_id_sets.instance_id_set[0] @@ -399,6 +432,7 @@ def provision(config: ProvisionConfig) -> int: config.repo, config.runner_uid, config.runner_gid, + enable_docker=config.enable_docker, ) ) @@ -450,6 +484,18 @@ def main() -> int: parser.add_argument("--security-group-id", default=os.environ.get("ALIYUN_ECS_SECURITY_GROUP_ID")) parser.add_argument("--image-id", default=os.environ.get("QUERY_REGRESSION_ECS_IMAGE_ID")) parser.add_argument("--instance-type", default=os.environ.get("ALIYUN_ECS_INSTANCE_TYPE")) + parser.add_argument( + "--system-disk-gib", + type=int, + default=os.environ.get("ALIYUN_ECS_SYSTEM_DISK_GIB", str(SYSTEM_DISK_GIB)), + ) + parser.add_argument( + "--ttl-hours", type=int, default=os.environ.get("ALIYUN_ECS_TTL_HOURS") or None + ) + parser.add_argument( + "--enable-docker", choices=("true", "false"), + default=os.environ.get("ALIYUN_ECS_ENABLE_DOCKER", "false"), + ) parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID")) parser.add_argument("--github-token", default=os.environ.get("GH_PERSONAL_ACCESS_TOKEN")) @@ -461,7 +507,7 @@ def main() -> int: missing = [ name for name, value in vars(args).items() - if name not in ("resource_group_id",) + if name not in ("resource_group_id", "ttl_hours") and (value is None or (isinstance(value, str) and not value)) ] if missing: @@ -480,6 +526,9 @@ def main() -> int: resource_group_id=args.resource_group_id or None, runner_uid=args.runner_uid, runner_gid=args.runner_gid, + system_disk_gib=args.system_disk_gib, + ttl_hours=args.ttl_hours, + enable_docker=args.enable_docker == "true", ) return provision(config) diff --git a/.github/scripts/aliyun-ecs-runner-teardown.py b/.github/scripts/aliyun-ecs-runner-teardown.py index 36c80253b9..658633e039 100644 --- a/.github/scripts/aliyun-ecs-runner-teardown.py +++ b/.github/scripts/aliyun-ecs-runner-teardown.py @@ -32,7 +32,8 @@ Two modes: runner by name. Both steps are idempotent and best-effort; a missing instance or runner is not an error. - Sweep (scheduled janitor): delete every instance tagged as managed by - query-regression CI whose creation time is older than the given TTL, and + query-regression CI whose creation time is older than its tagged TTL (or + the given fallback TTL for untagged instances), and deregister the matching runners. This is the safety net for runs whose teardown job never executed. """ @@ -72,16 +73,31 @@ def parse_creation_time(value: str) -> datetime: def expired_instance_names( - instances: list[tuple[str, str, str]], now: datetime, ttl: timedelta + instances: list[tuple[str, str, str, str | None]], now: datetime, ttl: timedelta ) -> list[tuple[str, str]]: """Pick (instance_id, instance_name) pairs whose creation is older than ttl. - `instances` items are (instance_id, instance_name, creation_time). + `instances` items are (instance_id, instance_name, creation_time, tagged_ttl_hours). """ expired = [] - for instance_id, instance_name, creation_time in instances: + for instance_id, instance_name, creation_time, tagged_ttl in instances: + instance_ttl = ttl + if tagged_ttl is not None: + try: + hours = int(tagged_ttl) + if not 1 <= hours <= 168: + raise ValueError("TTL outside 1..168 hours") + instance_ttl = timedelta(hours=hours) + except (TypeError, ValueError): + # Never fall back to a shorter TTL and kill a live tagged runner. + print( + f"Skipping {instance_id}: invalid {provision.TTL_TAG_KEY}={tagged_ttl!r}", + file=sys.stderr, + flush=True, + ) + continue age = now - parse_creation_time(creation_time) - if age >= ttl: + if age >= instance_ttl: expired.append((instance_id, instance_name)) return expired @@ -157,17 +173,20 @@ def deregister_runner(token: str, repo: str, runner_name: str) -> bool: return False -def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]]: +def list_managed_instances( + client, region_id: str +) -> list[tuple[str, str, str, str | None]]: from alibabacloud_ecs20140526 import models as ecs_models - result: list[tuple[str, str, str]] = [] + result: list[tuple[str, str, str, str | None]] = [] next_token = None while True: request = ecs_models.DescribeInstancesRequest( region_id=region_id, tag=[ ecs_models.DescribeInstancesRequestTag( - key=provision.MANAGED_BY_TAG_KEY, value=provision.MANAGED_BY_TAG_VALUE + key=provision.MANAGED_BY_TAG_KEY, + value=provision.MANAGED_BY_TAG_VALUE, ) ], max_results=100, @@ -175,7 +194,19 @@ def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]] ) response = client.describe_instances(request) for instance in response.body.instances.instance: - result.append((instance.instance_id, instance.instance_name, instance.creation_time)) + tags = (instance.tags.tag or []) if instance.tags else [] + tagged_ttl = next( + (tag.tag_value for tag in tags if tag.tag_key == provision.TTL_TAG_KEY), + None, + ) + result.append( + ( + instance.instance_id, + instance.instance_name, + instance.creation_time, + tagged_ttl, + ) + ) next_token = response.body.next_token if not next_token: return result @@ -187,7 +218,10 @@ def sweep(client, region_id: str, repo: str, github_token: str, ttl: timedelta) expired = expired_instance_names(instances, datetime.now(timezone.utc), ttl) ok = True for instance_id, instance_name in expired: - print(f"Instance {instance_id} ({instance_name}) exceeds TTL {ttl}; deleting", flush=True) + print( + f"Instance {instance_id} ({instance_name}) exceeds its configured TTL; deleting", + flush=True, + ) # Runner names mirror instance names by construction in the provision # script (both are qreg-ecs-). ok &= delete_instance(client, instance_id, region_id) diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml new file mode 100644 index 0000000000..665d73e727 --- /dev/null +++ b/.github/workflows/agent-observability.yml @@ -0,0 +1,324 @@ +name: Agent Observability Benchmark +on: + workflow_dispatch: + inputs: + targets: + description: all or comma-separated greptimedb,clickhouse,victorialogs + type: string + default: greptimedb + required: true + dataset: + description: 'Dataset size: S=5K, P=10M, M=100M' + type: choice + options: + - S + - P + - M + default: S + required: true + greptimedb_tag: + description: Docker Hub greptime/greptimedb tag + type: string + default: latest + required: true + clickhouse_tag: + description: Docker Hub clickhouse/clickhouse-server tag + type: string + default: 26.6.1.1193 + required: true + victorialogs_tag: + description: Docker Hub victoriametrics/victoria-logs tag or tag@sha256 digest + type: string + default: v1.52.0@sha256:47b820890d64c4575a2a0a46415dcd8a4fd59a0f1fcd6a377693d7aea639442e + required: true + o11ybench_ref: + description: Optional full commit SHA override (default uses runtime image revision) + type: string + required: false + runtime_image: + description: Optional Aliyun runtime image override (default uses repository variable) + type: string + required: false + ecs_instance_type: + description: ECS instance type (independent of Query Regression) + type: string + default: ecs.c9i.2xlarge + required: true + system_disk_gib: + description: ECS system disk size in GiB (20..2048; includes corpus, DB, images and swap) + type: number + default: 500 + required: true + benchmark_timeout_minutes: + description: Benchmark job timeout in minutes (1..360) + type: number + default: 360 + required: true + janitor_ttl_hours: + description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h) + type: number + default: 8 + required: true + db_cpus: + description: CPU limit per DB container + type: string + default: '4' + required: true + db_memory: + description: Memory limit per DB container + type: string + default: 8g + required: true +permissions: + contents: read +env: + TARGETS: ${{ inputs.targets }} + PROFILE: ${{ inputs.dataset }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} + CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }} + VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }} + O11YBENCH_REF: ${{ inputs.o11ybench_ref }} + RUNTIME_IMAGE: ${{ inputs.runtime_image || vars.O11YBENCH_RUNTIME_IMAGE }} + ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }} + SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} + BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }} + JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} + DB_CPUS: ${{ inputs.db_cpus }} + DB_MEMORY: ${{ inputs.db_memory }} +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + greptimedb: ${{ steps.inputs.outputs.greptimedb }} + clickhouse: ${{ steps.inputs.outputs.clickhouse }} + victorialogs: ${{ steps.inputs.outputs.victorialogs }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Test ECS lifecycle helpers + run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + - name: Validate requested benchmark + id: inputs + shell: bash + run: | + set -euo pipefail + # Validate cloud lifecycle budgets before provisioning; runtime owns workload validation. + [[ -n "$ECS_INSTANCE_TYPE" ]] + for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do + [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; } + done + ((SYSTEM_DISK_GIB >= 20 && SYSTEM_DISK_GIB <= 2048)) + ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168)) + # Reserve 45m provision + 15m teardown + 60m queue headroom before expiry. + ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) || { + echo 'Janitor TTL must cover benchmark timeout plus 2 hours' >&2; exit 1; + } + [[ -z "$O11YBENCH_REF" || "$O11YBENCH_REF" =~ ^[0-9a-f]{40}$ ]] + [[ -n "$RUNTIME_IMAGE" ]] || { echo 'Configure O11YBENCH_RUNTIME_IMAGE or supply runtime_image' >&2; exit 1; } + selected="$TARGETS" + [[ "$selected" != all ]] || selected=greptimedb,clickhouse,victorialogs + [[ "$selected" =~ ^(greptimedb|clickhouse|victorialogs)(,(greptimedb|clickhouse|victorialogs))*$ ]] + declare -A enabled + IFS=, read -ra targets <<< "$selected" + for target in "${targets[@]}"; do + [[ -z ${enabled[$target]:-} ]] || { echo "Duplicate target: $target" >&2; exit 1; } + enabled[$target]=true + done + for target in greptimedb clickhouse victorialogs; do + printf '%s=%s\n' "$target" "${enabled[$target]:-false}" >> "$GITHUB_OUTPUT" + done + provision: + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + label: ${{ steps.ecs.outputs.label }} + instance_id: ${{ steps.ecs.outputs.instance_id }} + runner_name: ${{ steps.ecs.outputs.runner_name }} + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Create ephemeral ECS runner + id: ecs + uses: ./greptimedb/.github/actions/aliyun-ecs-create + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} + security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} + image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} + instance-type: ${{ env.ECS_INSTANCE_TYPE }} + enable-docker: 'true' + system-disk-gib: ${{ inputs.system_disk_gib }} + ttl-hours: ${{ inputs.janitor_ttl_hours }} + resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} + runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} + runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} + benchmark: + needs: + - validate + - provision + runs-on: ${{ needs.provision.outputs.label }} + timeout-minutes: ${{ inputs.benchmark_timeout_minutes }} + env: + OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_ROOT: ${{ github.workspace }}/benchmark-data/${{ github.run_id }}-${{ github.run_attempt }} + MANIFEST_PATH: ${{ github.workspace }}/run-manifest.json + O11YBENCH_DIR: ${{ github.workspace }}/o11ybench + GREPTIMEDB_HTTP_PORT: '14000' + GREPTIMEDB_GRPC_PORT: '14001' + CLICKHOUSE_HTTP_PORT: '18123' + CLICKHOUSE_NATIVE_PORT: '19000' + VICTORIALOGS_HTTP_PORT: '19428' + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Check Docker on ephemeral Linux host + shell: bash + run: | + set -euo pipefail + [[ "$(uname -m)" == x86_64 ]] + jq --version + docker info + # The workspace (and Docker state) must not use a small /tmp filesystem. + df -h "$GITHUB_WORKSPACE" /var/lib/docker + - name: Resolve image tags and verify runtime revision + shell: bash + run: | + set -euo pipefail + actual= + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] + images='{}' + resolve_image() { + local target=$1 reference=$2 variable=$3 metadata id revision + docker pull "$reference" + metadata=$(docker image inspect "$reference") + id=$(jq -er '.[0].Id' <<< "$metadata") + if [[ "$target" == runtime ]]; then + revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") + [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || { echo 'Runtime lacks a full commit revision' >&2; return 1; } + [[ -z "$O11YBENCH_REF" || "$revision" == "$O11YBENCH_REF" ]] || { echo 'Runtime and requested o11ybench revisions differ' >&2; return 1; } + actual=$revision + printf 'RESOLVED_O11YBENCH_REF=%s\n' "$revision" >> "$GITHUB_ENV" + fi + printf '%s=%s\n' "$variable" "$id" >> "$GITHUB_ENV" + images=$(jq --arg target "$target" --arg requested "$reference" --argjson metadata "$metadata" \ + '. + {($target): {requested: $requested, id: $metadata[0].Id, digests: ($metadata[0].RepoDigests // [])}}' <<< "$images") + } + resolve_image runtime "$RUNTIME_IMAGE" RESOLVED_RUNTIME_IMAGE + for target in greptimedb clickhouse victorialogs; do + [[ "$TARGETS" == all || ",$TARGETS," == *",$target,"* ]] || continue + case "$target" in + greptimedb) resolve_image "$target" "greptime/greptimedb:$GREPTIMEDB_TAG" GREPTIMEDB_IMAGE ;; + clickhouse) resolve_image "$target" "clickhouse/clickhouse-server:$CLICKHOUSE_TAG" CLICKHOUSE_IMAGE ;; + victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;; + esac + done + jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \ + --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \ + --arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ + --argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \ + '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH" + - name: Checkout benchmark tools + uses: actions/checkout@v4 + with: + repository: GreptimeTeam/o11ybench + ref: ${{ env.RESOLVED_O11YBENCH_REF }} + path: o11ybench + persist-credentials: false + - name: Verify benchmark checkout revision + shell: bash + run: test "$(git -C "$O11YBENCH_DIR" rev-parse HEAD)" = "$RESOLVED_O11YBENCH_REF" + - name: Generate dataset once + id: generate + run: |- + bash o11ybench/scripts/run-agent-observability.sh --stage generate --execute \ + --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ + --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" + cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" + - name: Load and benchmark greptimedb + id: greptimedb + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} + run: |- + bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" + - name: Load and benchmark clickhouse + id: clickhouse + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }} + run: |- + bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target clickhouse --image "$CLICKHOUSE_IMAGE" \ + --clickhouse-http-port "$CLICKHOUSE_HTTP_PORT" --clickhouse-native-port "$CLICKHOUSE_NATIVE_PORT" + - name: Load and benchmark victorialogs + id: victorialogs + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victorialogs == 'true' }} + run: |- + bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target victorialogs --image "$VICTORIALOGS_IMAGE" \ + --victorialogs-http-port "$VICTORIALOGS_HTTP_PORT" + - name: Compare all three targets + if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome + == 'success' }} + run: |- + docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \ + --mount "type=bind,src=$ARTIFACT_ROOT,dst=/results" "$RESOLVED_RUNTIME_IMAGE" \ + scripts/finalize_logbench_agent_observability_measured.py \ + --greptimedb-summary /results/greptimedb-1/greptimedb-1/measured-summary.json \ + --clickhouse-summary /results/clickhouse-1/clickhouse-1/measured-summary.json \ + --victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \ + --output /results/comparison.json + - name: Clean up owned containers + if: always() + run: |- + if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then + bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + --owner "$OWNER" --root "$ARTIFACT_ROOT" + fi + - name: Upload results (no dataset) + if: always() + uses: actions/upload-artifact@v4 + with: + name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }} + path: |- + ${{ env.MANIFEST_PATH }} + ${{ env.ARTIFACT_ROOT }} + !${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl + if-no-files-found: warn + retention-days: 14 + teardown: + needs: + - provision + - benchmark + if: ${{ always() && needs.provision.outputs.instance_id != '' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Delete ECS and unregister runner + uses: ./greptimedb/.github/actions/aliyun-ecs-delete + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + instance-id: ${{ needs.provision.outputs.instance_id }} + runner-name: ${{ needs.provision.outputs.runner_name }} diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index c9db9ed5c5..8c9a93789c 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -157,25 +157,24 @@ jobs: ref: ${{ github.sha }} persist-credentials: false - - name: Install uv - uses: astral-sh/setup-uv@v6 - - name: Provision ECS runner id: provision - env: - ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} - ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} - GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }} - ALIYUN_ECS_VSWITCH_ID: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} - ALIYUN_ECS_SECURITY_GROUP_ID: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} - ALIYUN_ECS_INSTANCE_TYPE: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }} - ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} - QUERY_REGRESSION_ECS_IMAGE_ID: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} - QUERY_REGRESSION_RUNNER_UID: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} - QUERY_REGRESSION_RUNNER_GID: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} - run: >- - uv run .github/scripts/aliyun-ecs-runner-provision.py + uses: ./.github/actions/aliyun-ecs-create + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} + security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} + instance-type: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }} + # Preserve query regression's original resource and janitor budgets. + system-disk-gib: '80' + ttl-hours: '4' + resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} + image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} + runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} + runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} query-regression: needs: [provision, test-tooling] @@ -896,16 +895,12 @@ jobs: ref: ${{ github.sha }} persist-credentials: false - - name: Install uv - uses: astral-sh/setup-uv@v6 - - name: Teardown ECS runner - env: - ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} - ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} - GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }} - QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }} - QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }} - run: >- - uv run .github/scripts/aliyun-ecs-runner-teardown.py + uses: ./.github/actions/aliyun-ecs-delete + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + instance-id: ${{ needs.provision.outputs.instance_id }} + runner-name: ${{ needs.provision.outputs.runner_name }} diff --git a/tests/perf/test_aliyun_ecs_runner_scripts.py b/tests/perf/test_aliyun_ecs_runner_scripts.py index 581fc88abd..86d62fbf39 100644 --- a/tests/perf/test_aliyun_ecs_runner_scripts.py +++ b/tests/perf/test_aliyun_ecs_runner_scripts.py @@ -17,11 +17,16 @@ import base64 import importlib.util +import io +import os +import subprocess import sys import unittest +from contextlib import redirect_stderr from datetime import datetime, timedelta, timezone from pathlib import Path - +from types import SimpleNamespace +from unittest.mock import Mock, patch SCRIPTS_DIR = Path(__file__).parents[2] / ".github/scripts" @@ -35,8 +40,12 @@ def load_module(name: str, filename: str): return module -provision = load_module("aliyun_ecs_runner_provision_under_test", "aliyun-ecs-runner-provision.py") -teardown = load_module("aliyun_ecs_runner_teardown_under_test", "aliyun-ecs-runner-teardown.py") +provision = load_module( + "aliyun_ecs_runner_provision_under_test", "aliyun-ecs-runner-provision.py" +) +teardown = load_module( + "aliyun_ecs_runner_teardown_under_test", "aliyun-ecs-runner-teardown.py" +) class ProvisionNamingTest(unittest.TestCase): @@ -45,6 +54,127 @@ class ProvisionNamingTest(unittest.TestCase): self.assertEqual(provision.runner_label_for_run("12345"), "query-regression-ecs-12345") +class ProvisionResourcesTest(unittest.TestCase): + @staticmethod + def config(**overrides): + values = { + "region_id": "region", "vswitch_id": "vswitch", "security_group_id": "sg", + "image_id": "image", "instance_type": "ecs.u1-c1m1.2xlarge", "repo": "owner/repo", + "run_id": "12345", "github_token": "token", + } + return provision.ProvisionConfig(**(values | overrides)) + + def test_sdk_request_defaults_and_overrides(self): + models = SimpleNamespace( + RunInstancesRequest=SimpleNamespace, + RunInstancesRequestSystemDisk=SimpleNamespace, + RunInstancesRequestTag=SimpleNamespace, + ) + for overrides, disk, ttl in ( + ({}, "80", None), + ({"system_disk_gib": 500, "ttl_hours": 8}, "500", "8"), + ): + with ( + self.subTest(overrides=overrides), + patch.dict( + sys.modules, + {"alibabacloud_ecs20140526": SimpleNamespace(models=models)}, + ), + ): + client = Mock() + client.run_instances.return_value = SimpleNamespace( + body=SimpleNamespace( + instance_id_sets=SimpleNamespace(instance_id_set=["i-test"]) + ) + ) + self.assertEqual( + provision.run_instance( + client, self.config(**overrides), "userdata" + ), + "i-test", + ) + request = client.run_instances.call_args.args[0] + self.assertEqual(request.system_disk.size, disk) + self.assertEqual(request.system_disk.category, "cloud_essd") + self.assertEqual(request.instance_type, "ecs.u1-c1m1.2xlarge") + tags = {tag.key: tag.value for tag in request.tag} + expected = { + provision.MANAGED_BY_TAG_KEY: provision.MANAGED_BY_TAG_VALUE, + provision.RUN_TAG_KEY: "12345", + } + if ttl is not None: + expected[provision.TTL_TAG_KEY] = ttl + self.assertEqual(tags, expected) + + def test_invalid_resource_bounds(self): + for overrides in ( + {"system_disk_gib": 19}, + {"system_disk_gib": 2049}, + {"ttl_hours": 0}, + {"ttl_hours": 169}, + ): + with self.subTest(overrides=overrides), self.assertRaises(ValueError): + self.config(**overrides) + for disk in (20, 2048): + self.assertEqual(self.config(system_disk_gib=disk).system_disk_gib, disk) + for ttl in (1, 168): + self.assertEqual(self.config(ttl_hours=ttl).ttl_hours, ttl) + + def test_cli_environment_defaults_and_overrides(self): + args = [ + "provision", + "--region-id", + "region", + "--vswitch-id", + "vswitch", + "--security-group-id", + "sg", + "--image-id", + "image", + "--instance-type", + "ecs.u1-c1m1.2xlarge", + "--repo", + "owner/repo", + "--run-id", + "12345", + "--github-token", + "token", + ] + cases = [ + ({}, [], 80, None, False), + ( + {"ALIYUN_ECS_SYSTEM_DISK_GIB": "500", "ALIYUN_ECS_TTL_HOURS": "8"}, + [], + 500, + 8, + False, + ), + ( + {"ALIYUN_ECS_SYSTEM_DISK_GIB": "500", "ALIYUN_ECS_TTL_HOURS": "8"}, + ["--system-disk-gib", "600", "--ttl-hours", "12"], + 600, + 12, + False, + ), + ({"ALIYUN_ECS_TTL_HOURS": ""}, [], 80, None, False), + ({"ALIYUN_ECS_ENABLE_DOCKER": "true"}, [], 80, None, True), + ({"ALIYUN_ECS_ENABLE_DOCKER": "true"}, ["--enable-docker", "false"], 80, None, False), + ({}, ["--enable-docker", "true"], 80, None, True), + ] + for env, extra, disk, ttl, docker in cases: + with ( + self.subTest(env=env, extra=extra), + patch.dict(os.environ, env, clear=True), + patch.object(sys, "argv", args + extra), + patch.object(provision, "provision", return_value=0) as run, + ): + self.assertEqual(provision.main(), 0) + config = run.call_args.args[0] + self.assertEqual( + (config.system_disk_gib, config.ttl_hours, config.enable_docker), (disk, ttl, docker) + ) + + class ProvisionUserDataTest(unittest.TestCase): def render(self) -> str: return provision.render_user_data( @@ -71,6 +201,40 @@ class ProvisionUserDataTest(unittest.TestCase): self.assertIn("PATH=/opt/cargo/bin:", script) self.assertIn("systemctl restart --no-block ephemeral-github-runner.service", script) + def test_docker_setup_is_opt_in_and_precedes_runner(self): + args = ("runner-name", "label", "token", "owner/repo") + default = provision.render_user_data(*args) + self.assertEqual(default, provision.render_user_data(*args, enable_docker=False)) + self.assertNotIn("systemctl start docker", default) + self.assertNotIn("usermod", default) + enabled = provision.render_user_data(*args, runner_uid="2001", enable_docker=True) + start = enabled.index("# Reuse Docker CE") + end = enabled.index("cat > /etc/ephemeral-github-runner.env") + setup = enabled[start:end] + self.assertEqual(enabled[:start] + enabled[end:], + provision.render_user_data(*args, runner_uid="2001")) + self.assertLess(end, enabled.index("systemctl restart --no-block ephemeral-github-runner.service")) + for forbidden in ("apt-get", "sudo", "setfacl"): + self.assertNotIn(forbidden, setup) + mocks = ''' +docker() { :; } +jq() { :; } +systemctl() { echo "systemctl $*"; return "$FAIL_START"; } +id() { [[ "$*" == "-nu 2001" ]] || return 1; echo custom-runner; } +usermod() { echo "usermod $*"; } +runuser() { echo "runuser $*"; } +''' + for fail in ("0", "1"): + with self.subTest(fail_start=fail): + result = subprocess.run(["bash", "-euc", mocks + setup], + env=os.environ | {"FAIL_START": fail}, + capture_output=True, text=True) + self.assertEqual(result.returncode, int(fail), result.stderr) + calls = result.stdout.splitlines() + self.assertEqual(calls, ["docker", "jq", "systemctl start docker"] + ( + ["usermod -aG docker custom-runner", "runuser -u custom-runner -- docker info"] + if fail == "0" else [])) + def test_encode_user_data_round_trips(self) -> None: script = self.render() self.assertEqual( @@ -108,15 +272,100 @@ class TeardownExpiryTest(unittest.TestCase): def test_expired_instance_names_selects_only_old_instances(self) -> None: instances = [ - ("i-old", "qreg-ecs-1", "2026-08-17T01:00Z"), # 5h old: expired - ("i-edge", "qreg-ecs-2", "2026-08-17T02:00Z"), # exactly TTL: expired - ("i-fresh", "qreg-ecs-3", "2026-08-17T05:30Z"), # 30m old: kept + ("i-old", "qreg-ecs-1", "2026-08-17T01:00Z", None), # 5h old: expired + ("i-edge", "qreg-ecs-2", "2026-08-17T02:00Z", None), # exactly TTL: expired + ("i-fresh", "qreg-ecs-3", "2026-08-17T05:30Z", None), # 30m old: kept ] self.assertEqual( teardown.expired_instance_names(instances, self.NOW, self.TTL), [("i-old", "qreg-ecs-1"), ("i-edge", "qreg-ecs-2")], ) + def test_tagged_ttl_overrides_fallback_and_preserves_legacy(self): + instances = [ + ("i-legacy", "legacy", "2026-08-17T01:00Z", None), + ("i-long", "long", "2026-08-17T01:00Z", "8"), + ("i-short", "short", "2026-08-17T03:00Z", "2"), + ("i-edge", "edge", "2026-08-16T22:00Z", "8"), + ("i-not-yet", "not-yet", "2026-08-16T22:00:01Z", "8"), + ] + self.assertEqual( + teardown.expired_instance_names(instances, self.NOW, self.TTL), + [("i-legacy", "legacy"), ("i-short", "short"), ("i-edge", "edge")], + ) + + def test_malformed_ttl_never_falls_back_to_earlier_deletion(self): + for ttl in ("", "bad", "-1", "0", "169", "8.5", "nan"): + with self.subTest(ttl=ttl), redirect_stderr(io.StringIO()) as logs: + self.assertEqual( + teardown.expired_instance_names( + [("i-live", "live", "2026-08-16T01:00Z", ttl)], + self.NOW, + self.TTL, + ), + [], + ) + self.assertIn("Skipping i-live", logs.getvalue()) + + def test_list_instances_preserves_tags_and_pagination(self): + models = SimpleNamespace( + DescribeInstancesRequest=SimpleNamespace, + DescribeInstancesRequestTag=SimpleNamespace, + ) + client = Mock() + + def response(instance, token): + return SimpleNamespace( + body=SimpleNamespace( + instances=SimpleNamespace(instance=[instance]), next_token=token + ) + ) + + client.describe_instances.side_effect = [ + response( + SimpleNamespace( + instance_id="i-1", + instance_name="one", + creation_time="2026-08-17T01:00Z", + tags=SimpleNamespace( + tag=[ + SimpleNamespace( + tag_key=provision.TTL_TAG_KEY, tag_value="8" + ) + ] + ), + ), + "next", + ), + response( + SimpleNamespace( + instance_id="i-2", + instance_name="two", + creation_time="2026-08-17T02:00Z", + tags=None, + ), + None, + ), + ] + with patch.dict( + sys.modules, {"alibabacloud_ecs20140526": SimpleNamespace(models=models)} + ): + self.assertEqual( + teardown.list_managed_instances(client, "region"), + [ + ("i-1", "one", "2026-08-17T01:00Z", "8"), + ("i-2", "two", "2026-08-17T02:00Z", None), + ], + ) + self.assertEqual( + client.describe_instances.call_args_list[1].args[0].next_token, "next" + ) + tag = client.describe_instances.call_args_list[0].args[0].tag[0] + self.assertEqual( + (tag.key, tag.value), + (provision.MANAGED_BY_TAG_KEY, provision.MANAGED_BY_TAG_VALUE), + ) + if __name__ == "__main__": unittest.main()