ci: add manual agent observability benchmarks on Aliyun ECS (#9179)

* ci: add manual agent observability benchmarks on Aliyun ECS

Signed-off-by: WenyXu <wenymedia@gmail.com>

* ci: configure observability ECS budgets and reuse runner actions

Signed-off-by: WenyXu <wenymedia@gmail.com>

* ci: default observability runners to ecs.c9i.2xlarge

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(ci): prepare observability Docker access during ECS bootstrap

Signed-off-by: WenyXu <wenymedia@gmail.com>

* docs: remove standalone observability CI guide

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>

.github/actions/aliyun-ecs-create/README.md

@@ -0,0 +1,30 @@
+# Aliyun ECS runners
+
+Use `aliyun-ecs-create` to create an ECS runner and `aliyun-ecs-delete` to
+remove the instance and unregister the runner. Both actions run on GitHub-hosted
+Linux runners after checkout. Pass credentials through action inputs.
+
+Split the workflow into three jobs:
+
+1. Create the runner from a prepared ECS image.
+2. Run the workload with `runs-on: ${{ needs.create.outputs.label }}`.
+3. Delete the runner in a job with `if: always()` and dependencies on both jobs.
+   Pass `instance_id` and `runner_name` from the create job's outputs, even if
+   provisioning fails after creating the instance.
+
+See [agent-observability.yml](../../workflows/agent-observability.yml) for a caller
+and [action.yml](action.yml) for all inputs and outputs.
+
+`system-disk-gib` defaults to 80. Optional `ttl-hours` sets when the janitor may
+remove the instance, measured from ECS creation. Without it, the janitor uses
+its four-hour fallback. Allow time for provisioning, queueing, the workload and
+teardown. Normal teardown does not wait for the TTL.
+
+Deploy the TTL-aware janitor to the default branch before using a TTL above
+four hours; the old janitor ignores this setting. The actions retain the existing
+query-regression ownership tags and runner naming.
+
+Set `enable-docker: 'true'` for container workloads. Root cloud-init starts the
+image-provided Docker CE service and adds the runner to the Docker group before
+starting the runner. Docker group access is root-equivalent. This option defaults
+to `false`, so existing query-regression jobs keep their current permissions.

.github/actions/aliyun-ecs-create/action.yml

@@ -0,0 +1,85 @@
+name: Create Aliyun ECS runner
+description: Reuse the ephemeral runner lifecycle, including existing janitor ownership
+  tags.
+inputs:
+  access-key-id:
+    description: Aliyun access key ID
+    required: true
+  access-key-secret:
+    description: Aliyun access key secret
+    required: true
+  github-token:
+    description: Token authorized to register and remove repository runners
+    required: true
+  region-id:
+    description: ECS region
+    required: true
+  vswitch-id:
+    description: ECS VSwitch
+    required: true
+  security-group-id:
+    description: ECS security group
+    required: true
+  image-id:
+    description: Prepared ephemeral runner ECS image
+    required: true
+  instance-type:
+    description: ECS instance type
+    required: true
+  system-disk-gib:
+    description: System disk size in GiB (20..2048)
+    required: false
+    default: '80'
+  ttl-hours:
+    description: Optional per-instance janitor TTL (1..168 hours); empty preserves the janitor fallback
+    required: false
+    default: ''
+  enable-docker:
+    description: Start image-provided Docker and grant the runner Docker group access (root-equivalent)
+    required: false
+    default: 'false'
+  resource-group-id:
+    description: Optional ECS resource group
+    required: false
+    default: ''
+  runner-uid:
+    description: Runner user ID baked into the image
+    required: false
+    default: '1001'
+  runner-gid:
+    description: Runner group ID baked into the image
+    required: false
+    default: '1001'
+outputs:
+  label:
+    description: label
+    value: ${{ steps.provision.outputs.label }}
+  instance_id:
+    description: instance_id
+    value: ${{ steps.provision.outputs.instance_id }}
+  runner_name:
+    description: runner_name
+    value: ${{ steps.provision.outputs.runner_name }}
+runs:
+  using: composite
+  steps:
+  - uses: astral-sh/setup-uv@v6
+  - id: provision
+    shell: bash
+    env:
+      ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
+      ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }}
+      ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }}
+      ALIYUN_ECS_VSWITCH_ID: ${{ inputs.vswitch-id }}
+      ALIYUN_ECS_SECURITY_GROUP_ID: ${{ inputs.security-group-id }}
+      QUERY_REGRESSION_ECS_IMAGE_ID: ${{ inputs.image-id }}
+      ALIYUN_ECS_INSTANCE_TYPE: ${{ inputs.instance-type }}
+      ALIYUN_ECS_SYSTEM_DISK_GIB: ${{ inputs.system-disk-gib }}
+      ALIYUN_ECS_TTL_HOURS: ${{ inputs.ttl-hours }}
+      ALIYUN_ECS_ENABLE_DOCKER: ${{ inputs.enable-docker }}
+      ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ inputs.resource-group-id }}
+      QUERY_REGRESSION_RUNNER_UID: ${{ inputs.runner-uid }}
+      QUERY_REGRESSION_RUNNER_GID: ${{ inputs.runner-gid }}
+      ACTION_PATH: ${{ github.action_path }}
+    run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-provision.py"

.github/actions/aliyun-ecs-delete/action.yml

@@ -0,0 +1,37 @@
+name: Delete Aliyun ECS runner
+description: Reuse the ephemeral runner lifecycle, including existing janitor ownership
+  tags.
+inputs:
+  access-key-id:
+    description: Aliyun access key ID
+    required: true
+  access-key-secret:
+    description: Aliyun access key secret
+    required: true
+  github-token:
+    description: Token authorized to register and remove repository runners
+    required: true
+  region-id:
+    description: ECS region
+    required: true
+  instance-id:
+    description: Instance ID returned by create
+    required: true
+  runner-name:
+    description: Runner name returned by create
+    required: true
+runs:
+  using: composite
+  steps:
+  - uses: astral-sh/setup-uv@v6
+  - id: teardown
+    shell: bash
+    env:
+      ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
+      ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ inputs.access-key-secret }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ inputs.github-token }}
+      ALIYUN_ECS_REGION_ID: ${{ inputs.region-id }}
+      QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ inputs.instance-id }}
+      QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ inputs.runner-name }}
+      ACTION_PATH: ${{ github.action_path }}
+    run: uv run "$ACTION_PATH/../../scripts/aliyun-ecs-runner-teardown.py"

.github/scripts/aliyun-ecs-runner-provision.py

@@ -58,6 +58,7 @@ RUNNER_LABEL_PREFIX = "query-regression-ecs"
 MANAGED_BY_TAG_KEY = "managed-by"
 MANAGED_BY_TAG_VALUE = "query-regression-ci"
 RUN_TAG_KEY = "query-regression-run-id"
+TTL_TAG_KEY = "runner-ttl-hours"
 
 # Runner cache paths on the instance system disk. They are created empty
 # every provision and deleted with the VM.
@@ -100,6 +101,15 @@ class ProvisionConfig:
     # Runner identity inside the image; the workflow asserts the same values.
     runner_uid: str = "1001"
     runner_gid: str = "1001"
+    system_disk_gib: int = SYSTEM_DISK_GIB
+    ttl_hours: int | None = None
+    enable_docker: bool = False
+
+    def __post_init__(self) -> None:
+        if not 20 <= self.system_disk_gib <= 2048:
+            raise ValueError("system-disk-gib must be between 20 and 2048")
+        if self.ttl_hours is not None and not 1 <= self.ttl_hours <= 168:
+            raise ValueError("ttl-hours must be between 1 and 168")
 
 
 def runner_name_for_run(run_id: str) -> str:
@@ -117,8 +127,20 @@ def render_user_data(
     repo: str,
     runner_uid: str = "1001",
     runner_gid: str = "1001",
+    enable_docker: bool = False,
 ) -> str:
     """Render the cloud-init shell script for the runner instance."""
+    docker_setup = ""
+    if enable_docker:
+        docker_setup = f'''# Reuse Docker CE from the ECS image; run before dropping runner privileges.
+command -v docker
+command -v jq
+systemctl start docker
+runner_user=$(id -nu {runner_uid})
+usermod -aG docker "$runner_user"
+runuser -u "$runner_user" -- docker info
+
+'''
     destinations = " ".join(f'"{dst}"' for dst in CACHE_PATHS)
     cache_setup = f"""# Caches live on the system disk, are deleted with the instance, and every
 # run compiles cold. Within-run reuse (base warming candidate via the shared
@@ -158,7 +180,7 @@ set -euo pipefail
 
 {swap_setup}
 
-cat > /etc/ephemeral-github-runner.env <<'ENVEOF'
+{docker_setup}cat > /etc/ephemeral-github-runner.env <<'ENVEOF'
 RUNNER_NAME={runner_name}
 RUNNER_LABELS={runner_label}
 RUNNER_TOKEN={runner_token}
@@ -372,13 +394,24 @@ def run_instance(client, config: ProvisionConfig, user_data: str) -> str:
         internet_max_bandwidth_out=100,
         system_disk=ecs_models.RunInstancesRequestSystemDisk(
             category="cloud_essd",
-            size=str(SYSTEM_DISK_GIB),
+            size=str(config.system_disk_gib),
         ),
         user_data=user_data,
         tag=[
-            ecs_models.RunInstancesRequestTag(key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE),
+            ecs_models.RunInstancesRequestTag(
+                key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE
+            ),
             ecs_models.RunInstancesRequestTag(key=RUN_TAG_KEY, value=config.run_id),
-        ],
+        ]
+        + (
+            [
+                ecs_models.RunInstancesRequestTag(
+                    key=TTL_TAG_KEY, value=str(config.ttl_hours)
+                )
+            ]
+            if config.ttl_hours is not None
+            else []
+        ),
     )
     response = client.run_instances(request)
     instance_id = response.body.instance_id_sets.instance_id_set[0]
@@ -399,6 +432,7 @@ def provision(config: ProvisionConfig) -> int:
             config.repo,
             config.runner_uid,
             config.runner_gid,
+            enable_docker=config.enable_docker,
         )
     )
 
@@ -450,6 +484,18 @@ def main() -> int:
     parser.add_argument("--security-group-id", default=os.environ.get("ALIYUN_ECS_SECURITY_GROUP_ID"))
     parser.add_argument("--image-id", default=os.environ.get("QUERY_REGRESSION_ECS_IMAGE_ID"))
     parser.add_argument("--instance-type", default=os.environ.get("ALIYUN_ECS_INSTANCE_TYPE"))
+    parser.add_argument(
+        "--system-disk-gib",
+        type=int,
+        default=os.environ.get("ALIYUN_ECS_SYSTEM_DISK_GIB", str(SYSTEM_DISK_GIB)),
+    )
+    parser.add_argument(
+        "--ttl-hours", type=int, default=os.environ.get("ALIYUN_ECS_TTL_HOURS") or None
+    )
+    parser.add_argument(
+        "--enable-docker", choices=("true", "false"),
+        default=os.environ.get("ALIYUN_ECS_ENABLE_DOCKER", "false"),
+    )
     parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY"))
     parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID"))
     parser.add_argument("--github-token", default=os.environ.get("GH_PERSONAL_ACCESS_TOKEN"))
@@ -461,7 +507,7 @@ def main() -> int:
     missing = [
         name
         for name, value in vars(args).items()
-        if name not in ("resource_group_id",)
+        if name not in ("resource_group_id", "ttl_hours")
         and (value is None or (isinstance(value, str) and not value))
     ]
     if missing:
@@ -480,6 +526,9 @@ def main() -> int:
         resource_group_id=args.resource_group_id or None,
         runner_uid=args.runner_uid,
         runner_gid=args.runner_gid,
+        system_disk_gib=args.system_disk_gib,
+        ttl_hours=args.ttl_hours,
+        enable_docker=args.enable_docker == "true",
     )
     return provision(config)
 

.github/scripts/aliyun-ecs-runner-teardown.py

@@ -32,7 +32,8 @@ Two modes:
   runner by name. Both steps are idempotent and best-effort; a missing
   instance or runner is not an error.
 - Sweep (scheduled janitor): delete every instance tagged as managed by
-  query-regression CI whose creation time is older than the given TTL, and
+  query-regression CI whose creation time is older than its tagged TTL (or
+  the given fallback TTL for untagged instances), and
   deregister the matching runners. This is the safety net for runs whose
   teardown job never executed.
 """
@@ -72,16 +73,31 @@ def parse_creation_time(value: str) -> datetime:
 
 
 def expired_instance_names(
-    instances: list[tuple[str, str, str]], now: datetime, ttl: timedelta
+    instances: list[tuple[str, str, str, str | None]], now: datetime, ttl: timedelta
 ) -> list[tuple[str, str]]:
     """Pick (instance_id, instance_name) pairs whose creation is older than ttl.
 
-    `instances` items are (instance_id, instance_name, creation_time).
+    `instances` items are (instance_id, instance_name, creation_time, tagged_ttl_hours).
     """
     expired = []
-    for instance_id, instance_name, creation_time in instances:
+    for instance_id, instance_name, creation_time, tagged_ttl in instances:
+        instance_ttl = ttl
+        if tagged_ttl is not None:
+            try:
+                hours = int(tagged_ttl)
+                if not 1 <= hours <= 168:
+                    raise ValueError("TTL outside 1..168 hours")
+                instance_ttl = timedelta(hours=hours)
+            except (TypeError, ValueError):
+                # Never fall back to a shorter TTL and kill a live tagged runner.
+                print(
+                    f"Skipping {instance_id}: invalid {provision.TTL_TAG_KEY}={tagged_ttl!r}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                continue
         age = now - parse_creation_time(creation_time)
-        if age >= ttl:
+        if age >= instance_ttl:
             expired.append((instance_id, instance_name))
     return expired
 
@@ -157,17 +173,20 @@ def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
         return False
 
 
-def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]]:
+def list_managed_instances(
+    client, region_id: str
+) -> list[tuple[str, str, str, str | None]]:
     from alibabacloud_ecs20140526 import models as ecs_models
 
-    result: list[tuple[str, str, str]] = []
+    result: list[tuple[str, str, str, str | None]] = []
     next_token = None
     while True:
         request = ecs_models.DescribeInstancesRequest(
             region_id=region_id,
             tag=[
                 ecs_models.DescribeInstancesRequestTag(
-                    key=provision.MANAGED_BY_TAG_KEY, value=provision.MANAGED_BY_TAG_VALUE
+                    key=provision.MANAGED_BY_TAG_KEY,
+                    value=provision.MANAGED_BY_TAG_VALUE,
                 )
             ],
             max_results=100,
@@ -175,7 +194,19 @@ def list_managed_instances(client, region_id: str) -> list[tuple[str, str, str]]
         )
         response = client.describe_instances(request)
         for instance in response.body.instances.instance:
-            result.append((instance.instance_id, instance.instance_name, instance.creation_time))
+            tags = (instance.tags.tag or []) if instance.tags else []
+            tagged_ttl = next(
+                (tag.tag_value for tag in tags if tag.tag_key == provision.TTL_TAG_KEY),
+                None,
+            )
+            result.append(
+                (
+                    instance.instance_id,
+                    instance.instance_name,
+                    instance.creation_time,
+                    tagged_ttl,
+                )
+            )
         next_token = response.body.next_token
         if not next_token:
             return result
@@ -187,7 +218,10 @@ def sweep(client, region_id: str, repo: str, github_token: str, ttl: timedelta)
     expired = expired_instance_names(instances, datetime.now(timezone.utc), ttl)
     ok = True
     for instance_id, instance_name in expired:
-        print(f"Instance {instance_id} ({instance_name}) exceeds TTL {ttl}; deleting", flush=True)
+        print(
+            f"Instance {instance_id} ({instance_name}) exceeds its configured TTL; deleting",
+            flush=True,
+        )
         # Runner names mirror instance names by construction in the provision
         # script (both are qreg-ecs-<run_id>).
         ok &= delete_instance(client, instance_id, region_id)

.github/workflows/agent-observability.yml

@@ -0,0 +1,324 @@
+name: Agent Observability Benchmark
+on:
+  workflow_dispatch:
+    inputs:
+      targets:
+        description: all or comma-separated greptimedb,clickhouse,victorialogs
+        type: string
+        default: greptimedb
+        required: true
+      dataset:
+        description: 'Dataset size: S=5K, P=10M, M=100M'
+        type: choice
+        options:
+          - S
+          - P
+          - M
+        default: S
+        required: true
+      greptimedb_tag:
+        description: Docker Hub greptime/greptimedb tag
+        type: string
+        default: latest
+        required: true
+      clickhouse_tag:
+        description: Docker Hub clickhouse/clickhouse-server tag
+        type: string
+        default: 26.6.1.1193
+        required: true
+      victorialogs_tag:
+        description: Docker Hub victoriametrics/victoria-logs tag or tag@sha256 digest
+        type: string
+        default: v1.52.0@sha256:47b820890d64c4575a2a0a46415dcd8a4fd59a0f1fcd6a377693d7aea639442e
+        required: true
+      o11ybench_ref:
+        description: Optional full commit SHA override (default uses runtime image revision)
+        type: string
+        required: false
+      runtime_image:
+        description: Optional Aliyun runtime image override (default uses repository variable)
+        type: string
+        required: false
+      ecs_instance_type:
+        description: ECS instance type (independent of Query Regression)
+        type: string
+        default: ecs.c9i.2xlarge
+        required: true
+      system_disk_gib:
+        description: ECS system disk size in GiB (20..2048; includes corpus, DB, images and swap)
+        type: number
+        default: 500
+        required: true
+      benchmark_timeout_minutes:
+        description: Benchmark job timeout in minutes (1..360)
+        type: number
+        default: 360
+        required: true
+      janitor_ttl_hours:
+        description: Instance lifetime before janitor eligibility (1..168 hours; at least timeout + 2h)
+        type: number
+        default: 8
+        required: true
+      db_cpus:
+        description: CPU limit per DB container
+        type: string
+        default: '4'
+        required: true
+      db_memory:
+        description: Memory limit per DB container
+        type: string
+        default: 8g
+        required: true
+permissions:
+  contents: read
+env:
+  TARGETS: ${{ inputs.targets }}
+  PROFILE: ${{ inputs.dataset }}
+  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
+  CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
+  VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
+  O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image || vars.O11YBENCH_RUNTIME_IMAGE }}
+  ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
+  SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
+  BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
+  JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
+  DB_CPUS: ${{ inputs.db_cpus }}
+  DB_MEMORY: ${{ inputs.db_memory }}
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      greptimedb: ${{ steps.inputs.outputs.greptimedb }}
+      clickhouse: ${{ steps.inputs.outputs.clickhouse }}
+      victorialogs: ${{ steps.inputs.outputs.victorialogs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: greptimedb
+          persist-credentials: false
+      - name: Test ECS lifecycle helpers
+        run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
+      - name: Validate requested benchmark
+        id: inputs
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Validate cloud lifecycle budgets before provisioning; runtime owns workload validation.
+          [[ -n "$ECS_INSTANCE_TYPE" ]]
+          for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
+            [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; }
+          done
+          ((SYSTEM_DISK_GIB >= 20 && SYSTEM_DISK_GIB <= 2048))
+          ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
+          # Reserve 45m provision + 15m teardown + 60m queue headroom before expiry.
+          ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) || {
+            echo 'Janitor TTL must cover benchmark timeout plus 2 hours' >&2; exit 1;
+          }
+          [[ -z "$O11YBENCH_REF" || "$O11YBENCH_REF" =~ ^[0-9a-f]{40}$ ]]
+          [[ -n "$RUNTIME_IMAGE" ]] || { echo 'Configure O11YBENCH_RUNTIME_IMAGE or supply runtime_image' >&2; exit 1; }
+          selected="$TARGETS"
+          [[ "$selected" != all ]] || selected=greptimedb,clickhouse,victorialogs
+          [[ "$selected" =~ ^(greptimedb|clickhouse|victorialogs)(,(greptimedb|clickhouse|victorialogs))*$ ]]
+          declare -A enabled
+          IFS=, read -ra targets <<< "$selected"
+          for target in "${targets[@]}"; do
+            [[ -z ${enabled[$target]:-} ]] || { echo "Duplicate target: $target" >&2; exit 1; }
+            enabled[$target]=true
+          done
+          for target in greptimedb clickhouse victorialogs; do
+            printf '%s=%s\n' "$target" "${enabled[$target]:-false}" >> "$GITHUB_OUTPUT"
+          done
+  provision:
+    needs: validate
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    outputs:
+      label: ${{ steps.ecs.outputs.label }}
+      instance_id: ${{ steps.ecs.outputs.instance_id }}
+      runner_name: ${{ steps.ecs.outputs.runner_name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: greptimedb
+          persist-credentials: false
+      - name: Create ephemeral ECS runner
+        id: ecs
+        uses: ./greptimedb/.github/actions/aliyun-ecs-create
+        with:
+          access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+          access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+          vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
+          security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
+          image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
+          instance-type: ${{ env.ECS_INSTANCE_TYPE }}
+          enable-docker: 'true'
+          system-disk-gib: ${{ inputs.system_disk_gib }}
+          ttl-hours: ${{ inputs.janitor_ttl_hours }}
+          resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
+          runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
+          runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
+  benchmark:
+    needs:
+      - validate
+      - provision
+    runs-on: ${{ needs.provision.outputs.label }}
+    timeout-minutes: ${{ inputs.benchmark_timeout_minutes }}
+    env:
+      OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }}
+      ARTIFACT_ROOT: ${{ github.workspace }}/benchmark-data/${{ github.run_id }}-${{ github.run_attempt }}
+      MANIFEST_PATH: ${{ github.workspace }}/run-manifest.json
+      O11YBENCH_DIR: ${{ github.workspace }}/o11ybench
+      GREPTIMEDB_HTTP_PORT: '14000'
+      GREPTIMEDB_GRPC_PORT: '14001'
+      CLICKHOUSE_HTTP_PORT: '18123'
+      CLICKHOUSE_NATIVE_PORT: '19000'
+      VICTORIALOGS_HTTP_PORT: '19428'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: greptimedb
+          persist-credentials: false
+      - name: Check Docker on ephemeral Linux host
+        shell: bash
+        run: |
+          set -euo pipefail
+          [[ "$(uname -m)" == x86_64 ]]
+          jq --version
+          docker info
+          # The workspace (and Docker state) must not use a small /tmp filesystem.
+          df -h "$GITHUB_WORKSPACE" /var/lib/docker
+      - name: Resolve image tags and verify runtime revision
+        shell: bash
+        run: |
+          set -euo pipefail
+          actual=
+          [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
+          images='{}'
+          resolve_image() {
+            local target=$1 reference=$2 variable=$3 metadata id revision
+            docker pull "$reference"
+            metadata=$(docker image inspect "$reference")
+            id=$(jq -er '.[0].Id' <<< "$metadata")
+            if [[ "$target" == runtime ]]; then
+              revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
+              [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || { echo 'Runtime lacks a full commit revision' >&2; return 1; }
+              [[ -z "$O11YBENCH_REF" || "$revision" == "$O11YBENCH_REF" ]] || { echo 'Runtime and requested o11ybench revisions differ' >&2; return 1; }
+              actual=$revision
+              printf 'RESOLVED_O11YBENCH_REF=%s\n' "$revision" >> "$GITHUB_ENV"
+            fi
+            printf '%s=%s\n' "$variable" "$id" >> "$GITHUB_ENV"
+            images=$(jq --arg target "$target" --arg requested "$reference" --argjson metadata "$metadata" \
+              '. + {($target): {requested: $requested, id: $metadata[0].Id, digests: ($metadata[0].RepoDigests // [])}}' <<< "$images")
+          }
+          resolve_image runtime "$RUNTIME_IMAGE" RESOLVED_RUNTIME_IMAGE
+          for target in greptimedb clickhouse victorialogs; do
+            [[ "$TARGETS" == all || ",$TARGETS," == *",$target,"* ]] || continue
+            case "$target" in
+              greptimedb) resolve_image "$target" "greptime/greptimedb:$GREPTIMEDB_TAG" GREPTIMEDB_IMAGE ;;
+              clickhouse) resolve_image "$target" "clickhouse/clickhouse-server:$CLICKHOUSE_TAG" CLICKHOUSE_IMAGE ;;
+              victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
+            esac
+          done
+          jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
+            --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
+            --arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
+            --argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \
+            '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
+      - name: Checkout benchmark tools
+        uses: actions/checkout@v4
+        with:
+          repository: GreptimeTeam/o11ybench
+          ref: ${{ env.RESOLVED_O11YBENCH_REF }}
+          path: o11ybench
+          persist-credentials: false
+      - name: Verify benchmark checkout revision
+        shell: bash
+        run: test "$(git -C "$O11YBENCH_DIR" rev-parse HEAD)" = "$RESOLVED_O11YBENCH_REF"
+      - name: Generate dataset once
+        id: generate
+        run: |-
+          bash o11ybench/scripts/run-agent-observability.sh --stage generate --execute \
+            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
+            --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
+          cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+      - name: Load and benchmark greptimedb
+        id: greptimedb
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
+        run: |-
+          bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target greptimedb --image "$GREPTIMEDB_IMAGE" \
+            --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
+      - name: Load and benchmark clickhouse
+        id: clickhouse
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }}
+        run: |-
+          bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target clickhouse --image "$CLICKHOUSE_IMAGE" \
+            --clickhouse-http-port "$CLICKHOUSE_HTTP_PORT" --clickhouse-native-port "$CLICKHOUSE_NATIVE_PORT"
+      - name: Load and benchmark victorialogs
+        id: victorialogs
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victorialogs == 'true' }}
+        run: |-
+          bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target victorialogs --image "$VICTORIALOGS_IMAGE" \
+            --victorialogs-http-port "$VICTORIALOGS_HTTP_PORT"
+      - name: Compare all three targets
+        if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
+          == 'success' }}
+        run: |-
+          docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
+            --mount "type=bind,src=$ARTIFACT_ROOT,dst=/results" "$RESOLVED_RUNTIME_IMAGE" \
+            scripts/finalize_logbench_agent_observability_measured.py \
+            --greptimedb-summary /results/greptimedb-1/greptimedb-1/measured-summary.json \
+            --clickhouse-summary /results/clickhouse-1/clickhouse-1/measured-summary.json \
+            --victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \
+            --output /results/comparison.json
+      - name: Clean up owned containers
+        if: always()
+        run: |-
+          if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+            bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+              --owner "$OWNER" --root "$ARTIFACT_ROOT"
+          fi
+      - name: Upload results (no dataset)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |-
+            ${{ env.MANIFEST_PATH }}
+            ${{ env.ARTIFACT_ROOT }}
+            !${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl
+          if-no-files-found: warn
+          retention-days: 14
+  teardown:
+    needs:
+      - provision
+      - benchmark
+    if: ${{ always() && needs.provision.outputs.instance_id != '' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: greptimedb
+          persist-credentials: false
+      - name: Delete ECS and unregister runner
+        uses: ./greptimedb/.github/actions/aliyun-ecs-delete
+        with:
+          access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+          access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+          instance-id: ${{ needs.provision.outputs.instance_id }}
+          runner-name: ${{ needs.provision.outputs.runner_name }}

.github/workflows/query-regression.yml

@@ -157,25 +157,24 @@ jobs:
           ref: ${{ github.sha }}
           persist-credentials: false
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-
       - name: Provision ECS runner
         id: provision
-        env:
-          ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
-          ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
-          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }}
-          ALIYUN_ECS_VSWITCH_ID: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
-          ALIYUN_ECS_SECURITY_GROUP_ID: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
-          ALIYUN_ECS_INSTANCE_TYPE: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }}
-          ALIYUN_ECS_RESOURCE_GROUP_ID: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
-          QUERY_REGRESSION_ECS_IMAGE_ID: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
-          QUERY_REGRESSION_RUNNER_UID: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
-          QUERY_REGRESSION_RUNNER_GID: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
-        run: >-
-          uv run .github/scripts/aliyun-ecs-runner-provision.py
+        uses: ./.github/actions/aliyun-ecs-create
+        with:
+          access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+          access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+          vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
+          security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
+          instance-type: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }}
+          # Preserve query regression's original resource and janitor budgets.
+          system-disk-gib: '80'
+          ttl-hours: '4'
+          resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
+          image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
+          runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
+          runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
 
   query-regression:
     needs: [provision, test-tooling]
@@ -896,16 +895,12 @@ jobs:
           ref: ${{ github.sha }}
           persist-credentials: false
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-
       - name: Teardown ECS runner
-        env:
-          ALIBABA_CLOUD_ACCESS_KEY_ID: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
-          ALIBABA_CLOUD_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
-          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          ALIYUN_ECS_REGION_ID: ${{ vars.ALIYUN_ECS_REGION_ID }}
-          QUERY_REGRESSION_ECS_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
-          QUERY_REGRESSION_ECS_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
-        run: >-
-          uv run .github/scripts/aliyun-ecs-runner-teardown.py
+        uses: ./.github/actions/aliyun-ecs-delete
+        with:
+          access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
+          access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
+          instance-id: ${{ needs.provision.outputs.instance_id }}
+          runner-name: ${{ needs.provision.outputs.runner_name }}

tests/perf/test_aliyun_ecs_runner_scripts.py

@@ -17,11 +17,16 @@
 
 import base64
 import importlib.util
+import io
+import os
+import subprocess
 import sys
 import unittest
+from contextlib import redirect_stderr
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
 
 SCRIPTS_DIR = Path(__file__).parents[2] / ".github/scripts"
 
@@ -35,8 +40,12 @@ def load_module(name: str, filename: str):
     return module
 
 
-provision = load_module("aliyun_ecs_runner_provision_under_test", "aliyun-ecs-runner-provision.py")
-teardown = load_module("aliyun_ecs_runner_teardown_under_test", "aliyun-ecs-runner-teardown.py")
+provision = load_module(
+    "aliyun_ecs_runner_provision_under_test", "aliyun-ecs-runner-provision.py"
+)
+teardown = load_module(
+    "aliyun_ecs_runner_teardown_under_test", "aliyun-ecs-runner-teardown.py"
+)
 
 
 class ProvisionNamingTest(unittest.TestCase):
@@ -45,6 +54,127 @@ class ProvisionNamingTest(unittest.TestCase):
         self.assertEqual(provision.runner_label_for_run("12345"), "query-regression-ecs-12345")
 
 
+class ProvisionResourcesTest(unittest.TestCase):
+    @staticmethod
+    def config(**overrides):
+        values = {
+            "region_id": "region", "vswitch_id": "vswitch", "security_group_id": "sg",
+            "image_id": "image", "instance_type": "ecs.u1-c1m1.2xlarge", "repo": "owner/repo",
+            "run_id": "12345", "github_token": "token",
+        }
+        return provision.ProvisionConfig(**(values | overrides))
+
+    def test_sdk_request_defaults_and_overrides(self):
+        models = SimpleNamespace(
+            RunInstancesRequest=SimpleNamespace,
+            RunInstancesRequestSystemDisk=SimpleNamespace,
+            RunInstancesRequestTag=SimpleNamespace,
+        )
+        for overrides, disk, ttl in (
+            ({}, "80", None),
+            ({"system_disk_gib": 500, "ttl_hours": 8}, "500", "8"),
+        ):
+            with (
+                self.subTest(overrides=overrides),
+                patch.dict(
+                    sys.modules,
+                    {"alibabacloud_ecs20140526": SimpleNamespace(models=models)},
+                ),
+            ):
+                client = Mock()
+                client.run_instances.return_value = SimpleNamespace(
+                    body=SimpleNamespace(
+                        instance_id_sets=SimpleNamespace(instance_id_set=["i-test"])
+                    )
+                )
+                self.assertEqual(
+                    provision.run_instance(
+                        client, self.config(**overrides), "userdata"
+                    ),
+                    "i-test",
+                )
+                request = client.run_instances.call_args.args[0]
+                self.assertEqual(request.system_disk.size, disk)
+                self.assertEqual(request.system_disk.category, "cloud_essd")
+                self.assertEqual(request.instance_type, "ecs.u1-c1m1.2xlarge")
+                tags = {tag.key: tag.value for tag in request.tag}
+                expected = {
+                    provision.MANAGED_BY_TAG_KEY: provision.MANAGED_BY_TAG_VALUE,
+                    provision.RUN_TAG_KEY: "12345",
+                }
+                if ttl is not None:
+                    expected[provision.TTL_TAG_KEY] = ttl
+                self.assertEqual(tags, expected)
+
+    def test_invalid_resource_bounds(self):
+        for overrides in (
+            {"system_disk_gib": 19},
+            {"system_disk_gib": 2049},
+            {"ttl_hours": 0},
+            {"ttl_hours": 169},
+        ):
+            with self.subTest(overrides=overrides), self.assertRaises(ValueError):
+                self.config(**overrides)
+        for disk in (20, 2048):
+            self.assertEqual(self.config(system_disk_gib=disk).system_disk_gib, disk)
+        for ttl in (1, 168):
+            self.assertEqual(self.config(ttl_hours=ttl).ttl_hours, ttl)
+
+    def test_cli_environment_defaults_and_overrides(self):
+        args = [
+            "provision",
+            "--region-id",
+            "region",
+            "--vswitch-id",
+            "vswitch",
+            "--security-group-id",
+            "sg",
+            "--image-id",
+            "image",
+            "--instance-type",
+            "ecs.u1-c1m1.2xlarge",
+            "--repo",
+            "owner/repo",
+            "--run-id",
+            "12345",
+            "--github-token",
+            "token",
+        ]
+        cases = [
+            ({}, [], 80, None, False),
+            (
+                {"ALIYUN_ECS_SYSTEM_DISK_GIB": "500", "ALIYUN_ECS_TTL_HOURS": "8"},
+                [],
+                500,
+                8,
+                False,
+            ),
+            (
+                {"ALIYUN_ECS_SYSTEM_DISK_GIB": "500", "ALIYUN_ECS_TTL_HOURS": "8"},
+                ["--system-disk-gib", "600", "--ttl-hours", "12"],
+                600,
+                12,
+                False,
+            ),
+            ({"ALIYUN_ECS_TTL_HOURS": ""}, [], 80, None, False),
+            ({"ALIYUN_ECS_ENABLE_DOCKER": "true"}, [], 80, None, True),
+            ({"ALIYUN_ECS_ENABLE_DOCKER": "true"}, ["--enable-docker", "false"], 80, None, False),
+            ({}, ["--enable-docker", "true"], 80, None, True),
+        ]
+        for env, extra, disk, ttl, docker in cases:
+            with (
+                self.subTest(env=env, extra=extra),
+                patch.dict(os.environ, env, clear=True),
+                patch.object(sys, "argv", args + extra),
+                patch.object(provision, "provision", return_value=0) as run,
+            ):
+                self.assertEqual(provision.main(), 0)
+                config = run.call_args.args[0]
+                self.assertEqual(
+                    (config.system_disk_gib, config.ttl_hours, config.enable_docker), (disk, ttl, docker)
+                )
+
+
 class ProvisionUserDataTest(unittest.TestCase):
     def render(self) -> str:
         return provision.render_user_data(
@@ -71,6 +201,40 @@ class ProvisionUserDataTest(unittest.TestCase):
         self.assertIn("PATH=/opt/cargo/bin:", script)
         self.assertIn("systemctl restart --no-block ephemeral-github-runner.service", script)
 
+    def test_docker_setup_is_opt_in_and_precedes_runner(self):
+        args = ("runner-name", "label", "token", "owner/repo")
+        default = provision.render_user_data(*args)
+        self.assertEqual(default, provision.render_user_data(*args, enable_docker=False))
+        self.assertNotIn("systemctl start docker", default)
+        self.assertNotIn("usermod", default)
+        enabled = provision.render_user_data(*args, runner_uid="2001", enable_docker=True)
+        start = enabled.index("# Reuse Docker CE")
+        end = enabled.index("cat > /etc/ephemeral-github-runner.env")
+        setup = enabled[start:end]
+        self.assertEqual(enabled[:start] + enabled[end:],
+                         provision.render_user_data(*args, runner_uid="2001"))
+        self.assertLess(end, enabled.index("systemctl restart --no-block ephemeral-github-runner.service"))
+        for forbidden in ("apt-get", "sudo", "setfacl"):
+            self.assertNotIn(forbidden, setup)
+        mocks = '''
+docker() { :; }
+jq() { :; }
+systemctl() { echo "systemctl $*"; return "$FAIL_START"; }
+id() { [[ "$*" == "-nu 2001" ]] || return 1; echo custom-runner; }
+usermod() { echo "usermod $*"; }
+runuser() { echo "runuser $*"; }
+'''
+        for fail in ("0", "1"):
+            with self.subTest(fail_start=fail):
+                result = subprocess.run(["bash", "-euc", mocks + setup],
+                                        env=os.environ | {"FAIL_START": fail},
+                                        capture_output=True, text=True)
+                self.assertEqual(result.returncode, int(fail), result.stderr)
+                calls = result.stdout.splitlines()
+                self.assertEqual(calls, ["docker", "jq", "systemctl start docker"] + (
+                    ["usermod -aG docker custom-runner", "runuser -u custom-runner -- docker info"]
+                    if fail == "0" else []))
+
     def test_encode_user_data_round_trips(self) -> None:
         script = self.render()
         self.assertEqual(
@@ -108,15 +272,100 @@ class TeardownExpiryTest(unittest.TestCase):
 
     def test_expired_instance_names_selects_only_old_instances(self) -> None:
         instances = [
-            ("i-old", "qreg-ecs-1", "2026-08-17T01:00Z"),  # 5h old: expired
-            ("i-edge", "qreg-ecs-2", "2026-08-17T02:00Z"),  # exactly TTL: expired
-            ("i-fresh", "qreg-ecs-3", "2026-08-17T05:30Z"),  # 30m old: kept
+            ("i-old", "qreg-ecs-1", "2026-08-17T01:00Z", None),  # 5h old: expired
+            ("i-edge", "qreg-ecs-2", "2026-08-17T02:00Z", None),  # exactly TTL: expired
+            ("i-fresh", "qreg-ecs-3", "2026-08-17T05:30Z", None),  # 30m old: kept
         ]
         self.assertEqual(
             teardown.expired_instance_names(instances, self.NOW, self.TTL),
             [("i-old", "qreg-ecs-1"), ("i-edge", "qreg-ecs-2")],
         )
 
+    def test_tagged_ttl_overrides_fallback_and_preserves_legacy(self):
+        instances = [
+            ("i-legacy", "legacy", "2026-08-17T01:00Z", None),
+            ("i-long", "long", "2026-08-17T01:00Z", "8"),
+            ("i-short", "short", "2026-08-17T03:00Z", "2"),
+            ("i-edge", "edge", "2026-08-16T22:00Z", "8"),
+            ("i-not-yet", "not-yet", "2026-08-16T22:00:01Z", "8"),
+        ]
+        self.assertEqual(
+            teardown.expired_instance_names(instances, self.NOW, self.TTL),
+            [("i-legacy", "legacy"), ("i-short", "short"), ("i-edge", "edge")],
+        )
+
+    def test_malformed_ttl_never_falls_back_to_earlier_deletion(self):
+        for ttl in ("", "bad", "-1", "0", "169", "8.5", "nan"):
+            with self.subTest(ttl=ttl), redirect_stderr(io.StringIO()) as logs:
+                self.assertEqual(
+                    teardown.expired_instance_names(
+                        [("i-live", "live", "2026-08-16T01:00Z", ttl)],
+                        self.NOW,
+                        self.TTL,
+                    ),
+                    [],
+                )
+                self.assertIn("Skipping i-live", logs.getvalue())
+
+    def test_list_instances_preserves_tags_and_pagination(self):
+        models = SimpleNamespace(
+            DescribeInstancesRequest=SimpleNamespace,
+            DescribeInstancesRequestTag=SimpleNamespace,
+        )
+        client = Mock()
+
+        def response(instance, token):
+            return SimpleNamespace(
+                body=SimpleNamespace(
+                    instances=SimpleNamespace(instance=[instance]), next_token=token
+                )
+            )
+
+        client.describe_instances.side_effect = [
+            response(
+                SimpleNamespace(
+                    instance_id="i-1",
+                    instance_name="one",
+                    creation_time="2026-08-17T01:00Z",
+                    tags=SimpleNamespace(
+                        tag=[
+                            SimpleNamespace(
+                                tag_key=provision.TTL_TAG_KEY, tag_value="8"
+                            )
+                        ]
+                    ),
+                ),
+                "next",
+            ),
+            response(
+                SimpleNamespace(
+                    instance_id="i-2",
+                    instance_name="two",
+                    creation_time="2026-08-17T02:00Z",
+                    tags=None,
+                ),
+                None,
+            ),
+        ]
+        with patch.dict(
+            sys.modules, {"alibabacloud_ecs20140526": SimpleNamespace(models=models)}
+        ):
+            self.assertEqual(
+                teardown.list_managed_instances(client, "region"),
+                [
+                    ("i-1", "one", "2026-08-17T01:00Z", "8"),
+                    ("i-2", "two", "2026-08-17T02:00Z", None),
+                ],
+            )
+        self.assertEqual(
+            client.describe_instances.call_args_list[1].args[0].next_token, "next"
+        )
+        tag = client.describe_instances.call_args_list[0].args[0].tag[0]
+        self.assertEqual(
+            (tag.key, tag.value),
+            (provision.MANAGED_BY_TAG_KEY, provision.MANAGED_BY_TAG_VALUE),
+        )
+
 
 if __name__ == "__main__":
     unittest.main()