mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-28 22:56:19 +00:00
fix(ci): scope Aliyun janitor sweeps to the owning repository (#9966)
Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
@@ -80,8 +80,13 @@ after its TTL, so finish the inspection within that window.
|
||||
Trust model on the ECS path: the instance receives only the one-hour runner
|
||||
registration token via user data and holds no cloud credentials; the Aliyun
|
||||
AK/SK exist only in the control-plane jobs. Instance tags
|
||||
(`managed-by=query-regression-ci`, `query-regression-run-id`) feed the janitor
|
||||
sweep.
|
||||
(`managed-by=query-regression-ci`, `query-regression-run-id`,
|
||||
`github-repository`) feed the janitor sweep. The sweep only selects instances
|
||||
whose `github-repository` matches the requested repository. Older instances
|
||||
without that tag are excluded: verify their ownership before adding the tag,
|
||||
or clean them up by explicit instance ID. Per-run teardown remains unchanged.
|
||||
Update every repository sharing the account and region to the scoped helper;
|
||||
an older janitor can still sweep instances belonging to other repositories.
|
||||
|
||||
### Building and updating the ECS image
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ MANAGED_BY_TAG_KEY = "managed-by"
|
||||
MANAGED_BY_TAG_VALUE = "query-regression-ci"
|
||||
RUN_TAG_KEY = "query-regression-run-id"
|
||||
TTL_TAG_KEY = "runner-ttl-hours"
|
||||
REPO_TAG_KEY = "github-repository"
|
||||
|
||||
# Runner cache paths on the instance system disk. They are created empty
|
||||
# every provision and deleted with the VM.
|
||||
@@ -359,6 +360,7 @@ def run_instance(client, config: ProvisionConfig, user_data: str) -> str:
|
||||
key=MANAGED_BY_TAG_KEY, value=MANAGED_BY_TAG_VALUE
|
||||
),
|
||||
ecs_models.RunInstancesRequestTag(key=RUN_TAG_KEY, value=config.run_id),
|
||||
ecs_models.RunInstancesRequestTag(key=REPO_TAG_KEY, value=config.repo),
|
||||
]
|
||||
+ (
|
||||
[
|
||||
|
||||
@@ -32,8 +32,8 @@ Two modes:
|
||||
runner by name. Both steps are idempotent and best-effort; a missing
|
||||
instance or runner is not an error.
|
||||
- Sweep (scheduled janitor): delete every instance tagged as managed by
|
||||
query-regression CI whose creation time is older than its tagged TTL (or
|
||||
the given fallback TTL for untagged instances), and
|
||||
query-regression CI in the requested repository whose creation time is older
|
||||
than its tagged TTL (or the fallback TTL for instances without a TTL tag), and
|
||||
deregister the matching runners. This is the safety net for runs whose
|
||||
teardown job never executed.
|
||||
"""
|
||||
@@ -162,7 +162,7 @@ def delete_instance(client, instance_id: str, region_id: str | None = None) -> b
|
||||
|
||||
|
||||
def list_managed_instances(
|
||||
client, region_id: str
|
||||
client, region_id: str, repo: str
|
||||
) -> list[tuple[str, str, str, str | None]]:
|
||||
from alibabacloud_ecs20140526 import models as ecs_models
|
||||
|
||||
@@ -175,6 +175,9 @@ def list_managed_instances(
|
||||
ecs_models.DescribeInstancesRequestTag(
|
||||
key=provision.MANAGED_BY_TAG_KEY,
|
||||
value=provision.MANAGED_BY_TAG_VALUE,
|
||||
),
|
||||
ecs_models.DescribeInstancesRequestTag(
|
||||
key=provision.REPO_TAG_KEY, value=repo
|
||||
)
|
||||
],
|
||||
max_results=100,
|
||||
@@ -201,7 +204,7 @@ def list_managed_instances(
|
||||
|
||||
|
||||
def sweep(client, region_id: str, repo: str, github_token: str, ttl: timedelta) -> int:
|
||||
instances = list_managed_instances(client, region_id)
|
||||
instances = list_managed_instances(client, region_id, repo)
|
||||
print(f"Found {len(instances)} managed instance(s) in {region_id}", flush=True)
|
||||
expired = expired_instance_names(instances, datetime.now(timezone.utc), ttl)
|
||||
ok = True
|
||||
@@ -227,7 +230,7 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"--sweep",
|
||||
action="store_true",
|
||||
help="Janitor mode: delete all managed instances older than --sweep-ttl-hours.",
|
||||
help="Janitor mode: delete expired managed instances tagged for --repo.",
|
||||
)
|
||||
parser.add_argument("--sweep-ttl-hours", type=float, default=SWEEP_TTL.total_seconds() / 3600)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -143,6 +143,7 @@ class ProvisionResourcesTest(unittest.TestCase):
|
||||
expected = {
|
||||
provision.MANAGED_BY_TAG_KEY: provision.MANAGED_BY_TAG_VALUE,
|
||||
provision.RUN_TAG_KEY: "12345",
|
||||
"github-repository": "owner/repo",
|
||||
}
|
||||
if ttl is not None:
|
||||
expected[provision.TTL_TAG_KEY] = ttl
|
||||
@@ -423,7 +424,7 @@ class TeardownExpiryTest(unittest.TestCase):
|
||||
sys.modules, {"alibabacloud_ecs20140526": SimpleNamespace(models=models)}
|
||||
):
|
||||
self.assertEqual(
|
||||
teardown.list_managed_instances(client, "region"),
|
||||
teardown.list_managed_instances(client, "region", "owner/repo"),
|
||||
[
|
||||
("i-1", "one", "2026-08-17T01:00Z", "8"),
|
||||
("i-2", "two", "2026-08-17T02:00Z", None),
|
||||
@@ -432,11 +433,49 @@ class TeardownExpiryTest(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
client.describe_instances.call_args_list[1].args[0].next_token, "next"
|
||||
)
|
||||
tag = client.describe_instances.call_args_list[0].args[0].tag[0]
|
||||
self.assertEqual(
|
||||
(tag.key, tag.value),
|
||||
(provision.MANAGED_BY_TAG_KEY, provision.MANAGED_BY_TAG_VALUE),
|
||||
for call in client.describe_instances.call_args_list:
|
||||
self.assertEqual(
|
||||
{tag.key: tag.value for tag in call.args[0].tag},
|
||||
{"managed-by": "query-regression-ci", "github-repository": "owner/repo"},
|
||||
)
|
||||
|
||||
def test_sweep_isolates_repositories_and_untagged_instances(self):
|
||||
models = SimpleNamespace(
|
||||
DescribeInstancesRequest=SimpleNamespace,
|
||||
DescribeInstancesRequestTag=SimpleNamespace,
|
||||
)
|
||||
instances = [
|
||||
("own", {"managed-by": "query-regression-ci", "github-repository": "owner/repo"}),
|
||||
("other", {"managed-by": "query-regression-ci", "github-repository": "owner/other"}),
|
||||
("legacy", {"managed-by": "query-regression-ci"}),
|
||||
("unmanaged", {"github-repository": "owner/repo"}),
|
||||
]
|
||||
|
||||
def describe(request):
|
||||
# Model ECS tag intersection using the actual request filters.
|
||||
selected = [
|
||||
SimpleNamespace(instance_id=identity, instance_name=identity,
|
||||
creation_time="2020-01-01T00:00Z", tags=None)
|
||||
for identity, tags in instances
|
||||
if all(tags.get(tag.key) == tag.value for tag in request.tag)
|
||||
]
|
||||
return SimpleNamespace(body=SimpleNamespace(
|
||||
instances=SimpleNamespace(instance=selected), next_token=None))
|
||||
|
||||
with (
|
||||
patch.dict(sys.modules, {"alibabacloud_ecs20140526": SimpleNamespace(models=models)}),
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete,
|
||||
patch.object(teardown, "deregister_runner", return_value=True) as deregister,
|
||||
):
|
||||
client = Mock()
|
||||
client.describe_instances.side_effect = describe
|
||||
for repo, expected in (("owner/repo", "own"), ("owner/other", "other")):
|
||||
with self.subTest(repo=repo):
|
||||
delete.reset_mock()
|
||||
deregister.reset_mock()
|
||||
self.assertEqual(teardown.sweep(client, "region", repo, "token", self.TTL), 0)
|
||||
delete.assert_called_once_with(client, expected, "region")
|
||||
deregister.assert_called_once_with("token", repo, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user