From 6504af641ef3e2a6269c4fc64a3ff80e843766b0 Mon Sep 17 00:00:00 2001 From: localhost Date: Mon, 31 Aug 2026 06:18:20 +0000 Subject: [PATCH] fix: increase system disk size to 50 GiB for ECS instances (#8986) --- .github/runner-scale-sets/query-regression/README.md | 6 +++--- .github/scripts/aliyun-ecs-runner-provision.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md index ce447dbdd0..c0e34a4c40 100644 --- a/.github/runner-scale-sets/query-regression/README.md +++ b/.github/runner-scale-sets/query-regression/README.md @@ -49,7 +49,7 @@ Configuration lives in repository variables/secrets: | vars | `ALIYUN_ECS_INSTANCE_TYPE` | Dedicated (non-burstable, non-shared) instance family. Prefer 32 GiB (e.g. `ecs.g8i.2xlarge`); `ecs.c9i.2xlarge` is 8c16g and nightly thin-LTO of greptime peaks above that. Both base and candidate clusters run on the same machine, so noisy neighbors break thresholds. | | vars | `QUERY_REGRESSION_ECS_IMAGE_ID` | Custom image built by `ecs-image/build-ecs-image.py`. | -The system disk is 40 GiB, which covers the image, a 16 GiB swapfile, the +The system disk is 50 GiB, which covers the image, a 16 GiB swapfile, the checkout, and cold build caches (target dir, cargo registry, sccache). ENOSPC stops the runner itself from writing logs, which GitHub reports as `The operation was canceled` with no telemetry, indistinguishable from a @@ -164,7 +164,7 @@ reintroduce those actions unless the image contract changes. Each run provisions its own ECS instance, so overlapping dispatches proceed in parallel. There is no workflow `concurrency` group. All Cargo state (`CARGO_HOME` including registry/git, `CARGO_TARGET_DIR`, sccache, cache -metadata) lives on the 40 GiB system disk and is discarded with the VM. +metadata) lives on the 50 GiB system disk and is discarded with the VM. `RUSTUP_HOME=/opt/rustup` and `/opt/cargo/bin` are image-owned. The runner sets `RUSTC_WRAPPER=/usr/local/bin/sccache`, `SCCACHE_DIR=/home/runner/.cache/sccache`, `SCCACHE_CACHE_SIZE=10G`, and @@ -173,7 +173,7 @@ at 10G. Base and candidate builds share the target on that disk; Cargo fingerprints invalidate source and dependency changes. The workflow reports `du` / `df` in telemetry. It does not try to reclaim -space across runs: a cold 40 GiB disk that fills up fails the build. +space across runs: a cold 50 GiB disk that fills up fails the build. ## Future optional phases diff --git a/.github/scripts/aliyun-ecs-runner-provision.py b/.github/scripts/aliyun-ecs-runner-provision.py index 74a69681f3..307ee336eb 100644 --- a/.github/scripts/aliyun-ecs-runner-provision.py +++ b/.github/scripts/aliyun-ecs-runner-provision.py @@ -80,6 +80,9 @@ POLL_INTERVAL_SECONDS = 5 # to the cold build caches. SWAP_FILE = "/swapfile" SWAP_SIZE_GIB = 16 +# Image (~12G), 16G swap, checkout, base+candidate data homes, and cold +# build caches. Deleted with the instance. +SYSTEM_DISK_GIB = 50 @dataclass(frozen=True) @@ -367,10 +370,10 @@ def run_instance(client, config: ProvisionConfig, user_data: str) -> str: spot_strategy="NoSpot", internet_charge_type="PayByTraffic", internet_max_bandwidth_out=100, - # Holds the image (~12G), the 16G swapfile, the source checkout, the - # base+candidate cluster data homes, and cold build caches. Deleted - # with the instance. - system_disk=ecs_models.RunInstancesRequestSystemDisk(category="cloud_essd", size="40"), + system_disk=ecs_models.RunInstancesRequestSystemDisk( + category="cloud_essd", + size=str(SYSTEM_DISK_GIB), + ), user_data=user_data, tag=[ ecs_models.RunInstancesRequestTag(key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE),