From 7d9ea26530e4700a25c9f8a3754bb1ba853860c6 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 6 Jun 2024 20:26:57 +0100 Subject: [PATCH] CI(test_runner): Upload all test artifacts if preserve_database_files is enabled --- test_runner/fixtures/neon_fixtures.py | 10 +++++++++- test_runner/fixtures/utils.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index a25b8bfca1..9b0d22cb4e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1439,6 +1439,8 @@ def neon_env_builder( pageserver_default_tenant_config_compaction_algorithm=pageserver_default_tenant_config_compaction_algorithm, ) as builder: yield builder + if builder.preserve_database_files: + request.node.user_properties.append(("preserve_database_files", True)) @dataclass @@ -4100,7 +4102,13 @@ def test_output_dir( yield test_dir - allure_attach_from_dir(test_dir) + preserve_database_files = False + for k, v in request.node.user_properties: + if k == "preserve_database_files": + assert isinstance(v, bool) + preserve_database_files = v + + allure_attach_from_dir(test_dir, preserve_database_files) class FileAndThreadLock: diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index b55329e054..d06d2f3d93 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -240,9 +240,18 @@ ATTACHMENT_NAME_REGEX: re.Pattern = re.compile( # type: ignore[type-arg] ) -def allure_attach_from_dir(dir: Path): +def allure_attach_from_dir(dir: Path, preserve_database_files: bool = False): """Attach all non-empty files from `dir` that matches `ATTACHMENT_NAME_REGEX` to Allure report""" + if preserve_database_files: + zst_file = dir.with_suffix(".tar.zst") + with zst_file.open("wb") as zst: + cctx = zstandard.ZstdCompressor() + with cctx.stream_writer(zst) as compressor: + with tarfile.open(fileobj=compressor, mode="w") as tar: + tar.add(dir, arcname="") + allure.attach.file(zst_file, "everything.tar.zst", "application/zstd", "tar.zst") + for attachment in Path(dir).glob("**/*"): if ATTACHMENT_NAME_REGEX.fullmatch(attachment.name) and attachment.stat().st_size > 0: name = str(attachment.relative_to(dir))