diff --git a/ALEK_LIST_FILES.txt b/ALEK_LIST_FILES.txt index eb78122949..1545890adc 100644 --- a/ALEK_LIST_FILES.txt +++ b/ALEK_LIST_FILES.txt @@ -1 +1 @@ -[RemotePath("tenants/6810db134e89f13adf7c18bb28e01134/timelines/144233037159e56e413ee0fc57ad801c/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/6810db134e89f13adf7c18bb28e01134/timelines/144233037159e56e413ee0fc57ad801c/index_part.json"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/606d928c7fb4c43f78e2b1f01ea39a84/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/606d928c7fb4c43f78e2b1f01ea39a84/index_part.json"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/77a3b18b639675d46e53922bc4afc444/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/77a3b18b639675d46e53922bc4afc444/index_part.json"), RemotePath("v15/share/extension/test_ext.control")] \ No newline at end of file +[RemotePath("v15/share/extension/test_ext.control")] diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index f8b90de557..c2fc90d00f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -646,12 +646,14 @@ class NeonEnvBuilder: return env + # TODO WAN: make new thing def enable_remote_storage( self, remote_storage_kind: RemoteStorageKind, test_name: str, force_enable: bool = True, ): + # TODO: more of these? if remote_storage_kind == RemoteStorageKind.NOOP: return elif remote_storage_kind == RemoteStorageKind.LOCAL_FS: @@ -673,13 +675,14 @@ class NeonEnvBuilder: assert force_enable or self.remote_storage is None, "remote storage is enabled already" self.remote_storage = LocalFsStorage(Path(self.repo_dir / "local_fs_remote_storage")) - # TODO: do we need a new method here? for remote_ext_config because we are not the pageserver? - # or is it ok to re-use the s3 bucket? + # TODO: do we need a new method here? for remote_ext_config because we are not the pageserver? A: YES def enable_mock_s3_remote_storage(self, bucket_name: str, force_enable: bool = True): """ Sets up the pageserver to use the S3 mock server, creates the bucket, if it's not present already. Starts up the mock server, if that does not run yet. Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`. + + TODO: this also does stuff for ext_remote_storage bucket """ assert force_enable or self.remote_storage is None, "remote storage is enabled already" mock_endpoint = self.mock_s3_server.endpoint() @@ -702,6 +705,16 @@ class NeonEnvBuilder: secret_key=self.mock_s3_server.secret_key(), ) + ext_bucket_name = f"ext_{bucket_name}" + self.remote_storage_client.create_bucket(Bucket=ext_bucket_name) + self.ext_remote_storage = S3Storage( + bucket_name=ext_bucket_name, + endpoint=mock_endpoint, + bucket_region=mock_region, + access_key=self.mock_s3_server.access_key(), + secret_key=self.mock_s3_server.secret_key(), + ) + def enable_real_s3_remote_storage(self, test_name: str, force_enable: bool = True): """ Sets up configuration to use real s3 endpoint without mock server @@ -904,6 +917,8 @@ class NeonEnv: self.neon_binpath = config.neon_binpath self.pg_distrib_dir = config.pg_distrib_dir self.endpoint_counter = 0 + self.remote_storage_client = config.remote_storage_client + self.ext_remote_storage = config.ext_remote_storage # generate initial tenant ID here instead of letting 'neon init' generate it, # so that we don't need to dig it out of the config file afterwards. diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index 7b63783a07..6429888121 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -6,6 +6,8 @@ from fixtures.neon_fixtures import ( ) import json +TEST_EXT_PATH = "v15/share/extension/test_ext.control" + def test_file_download(neon_env_builder: NeonEnvBuilder): """ @@ -26,22 +28,17 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 3 env = neon_env_builder.init_start() - TEST_EXT_PATH = "v15/share/extension/test_ext.control" - - # TODO: we shouldn't be using neon_env_builder.remote_storage_client, - # we should pass the remote_storage_client to env in the builder. - # 4. Upload test_ext.control file to the bucket # In the non-mock version this is done by CI/CD with open("test_ext.control", "rb") as data: - neon_env_builder.remote_storage_client.upload_fileobj( - data, neon_env_builder.remote_storage.bucket_name, TEST_EXT_PATH + env.remote_storage_client.upload_fileobj( + data, env.ext_remote_storage.bucket_name, TEST_EXT_PATH ) # 5. Download file from the bucket to correct local location # Later this will be replaced by our rust code - resp = neon_env_builder.remote_storage_client.get_object( - Bucket=neon_env_builder.remote_storage.bucket_name, Key=TEST_EXT_PATH + resp = env.remote_storage_client.get_object( + Bucket=env.ext_remote_storage.bucket_name, Key=TEST_EXT_PATH ) response = resp["Body"] for pgres_version in ("v15", "v14"): @@ -54,9 +51,9 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): remote_ext_config = json.dumps( { - "bucket": neon_env_builder.remote_storage.bucket_name, + "bucket": env.ext_remote_storage.bucket_name, "region": "us-east-1", - "endpoint": neon_env_builder.remote_storage.endpoint, + "endpoint": env.ext_remote_storage.endpoint, } )