From 55371af71106b3b8a6ddb3231ed6d73bf18a8848 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 18 Sep 2023 10:24:53 +0300 Subject: [PATCH] test: workaround known bad mock_s3 ListObjectsV2 response (#5330) this should allow test test_delete_tenant_exercise_crash_safety_failpoints with debug-pg16-Check.RETRY_WITH_RESTART-mock_s3-tenant-delete-before-remove-timelines-dir-True to pass more reliably. --- test_runner/fixtures/pageserver/utils.py | 28 +++++++++++++++++------- test_runner/fixtures/remote_storage.py | 4 ++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 3197017bd1..2e5d75a0fc 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -236,15 +236,27 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str response = list_prefix(neon_env_builder, prefix) keys = response["KeyCount"] objects = response.get("Contents", []) + common_prefixes = response.get("CommonPrefixes", []) - if keys != 0 and len(objects) == 0: - # this has been seen in one case with mock_s3: - # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-4938/6000769714/index.html#suites/3556ed71f2d69272a7014df6dcb02317/ca01e4f4d8d9a11f - # looking at moto impl, it might be there's a race with common prefix (sub directory) not going away with deletes - common_prefixes = response.get("CommonPrefixes", []) - log.warn( - f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" - ) + remote_storage = neon_env_builder.pageserver_remote_storage + is_mock_s3 = isinstance(remote_storage, S3Storage) and not remote_storage.cleanup + + if is_mock_s3: + if keys == 1 and len(objects) == 0 and len(common_prefixes) == 1: + # this has been seen in the wild by tests with the below contradicting logging + # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5322/6207777020/index.html#suites/3556ed71f2d69272a7014df6dcb02317/53b5c368b5a68865 + # this seems like a mock_s3 issue + log.warn( + f"contrading ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}, assuming this means KeyCount=0" + ) + keys = 0 + elif keys != 0 and len(objects) == 0: + # this has been seen in one case with mock_s3: + # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-4938/6000769714/index.html#suites/3556ed71f2d69272a7014df6dcb02317/ca01e4f4d8d9a11f + # looking at moto impl, it might be there's a race with common prefix (sub directory) not going away with deletes + log.warn( + f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" + ) assert keys == 0, f"remote dir with prefix {prefix} is not empty after deletion: {objects}" diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py index ca7ad93f83..f7cddbc821 100644 --- a/test_runner/fixtures/remote_storage.py +++ b/test_runner/fixtures/remote_storage.py @@ -115,6 +115,8 @@ class S3Storage: prefix_in_bucket: str client: S3Client cleanup: bool + """Is this MOCK_S3 (false) or REAL_S3 (true)""" + real: bool endpoint: Optional[str] = None def access_env_vars(self) -> Dict[str, str]: @@ -265,6 +267,7 @@ class RemoteStorageKind(str, enum.Enum): prefix_in_bucket="", client=client, cleanup=False, + real=False, ) assert self == RemoteStorageKind.REAL_S3 @@ -300,6 +303,7 @@ class RemoteStorageKind(str, enum.Enum): prefix_in_bucket=prefix_in_bucket, client=client, cleanup=True, + real=True, )