From 0ac2606c8ac0b09859ce6b6a32e9e97066de0130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 2 Feb 2024 23:45:57 +0100 Subject: [PATCH] S3 restore test: Use a workaround to enable moto's self-copy support (#6594) While working on https://github.com/getmoto/moto/pull/7303 I discovered that if you enable bucket encryption, moto allows self-copies. So we can un-ignore the test. I tried it out locally, it works great. Followup of #6533, part of https://github.com/neondatabase/cloud/issues/8233 --- test_runner/fixtures/pageserver/utils.py | 18 +++++++++++++++++- test_runner/regress/test_s3_restore.py | 2 -- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 4cfdee6e01..c2281ae25a 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -356,10 +356,26 @@ def enable_remote_storage_versioning( """ Enable S3 versioning for the remote storage """ - # local_fs has no + # local_fs has no support for versioning assert isinstance(remote, S3Storage), "localfs is currently not supported" assert remote.client is not None + # The SDK supports enabling versioning on normal S3 as well but we don't want to change + # these settings from a test in a live bucket (also, our access isn't enough nor should it be) + assert not remote.real, "Enabling storage versioning only supported on Mock S3" + + # Workaround to enable self-copy until upstream bug is fixed: https://github.com/getmoto/moto/issues/7300 + remote.client.put_bucket_encryption( + Bucket=remote.bucket_name, + ServerSideEncryptionConfiguration={ + "Rules": [ + { + "ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}, + "BucketKeyEnabled": False, + }, + ] + }, + ) # Note that this doesnt use pagination, so list is not guaranteed to be exhaustive. response = remote.client.put_bucket_versioning( Bucket=remote.bucket_name, diff --git a/test_runner/regress/test_s3_restore.py b/test_runner/regress/test_s3_restore.py index 188d8a3b33..aaa33f0bcb 100644 --- a/test_runner/regress/test_s3_restore.py +++ b/test_runner/regress/test_s3_restore.py @@ -1,7 +1,6 @@ import time from datetime import datetime, timezone -import pytest from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, @@ -32,7 +31,6 @@ def test_tenant_s3_restore( remote_storage = neon_env_builder.pageserver_remote_storage assert remote_storage, "remote storage not configured" enable_remote_storage_versioning(remote_storage) - pytest.skip("moto doesn't support self-copy: https://github.com/getmoto/moto/issues/7300") env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) env.pageserver.allowed_errors.extend(