rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-15 17:32:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

separate bucket for extensions

Browse Source
This commit is contained in:
Alek Westover
2023-06-21 13:18:35 -04:00
parent 85a7511700
commit 5e1e859ab8
3 changed files with 26 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ALEK_LIST_FILES.txt
View File

@@ -1 +1 @@
[RemotePath("tenants/6810db134e89f13adf7c18bb28e01134/timelines/144233037159e56e413ee0fc57ad801c/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/6810db134e89f13adf7c18bb28e01134/timelines/144233037159e56e413ee0fc57ad801c/index_part.json"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/606d928c7fb4c43f78e2b1f01ea39a84/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/606d928c7fb4c43f78e2b1f01ea39a84/index_part.json"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/77a3b18b639675d46e53922bc4afc444/000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__000000000169B098-000000000169B111"), RemotePath("tenants/ba2b4f2dd797a534f5cbed833a829c0a/timelines/77a3b18b639675d46e53922bc4afc444/index_part.json"), RemotePath("v15/share/extension/test_ext.control")]
[RemotePath("v15/share/extension/test_ext.control")]

19
test_runner/fixtures/neon_fixtures.py
View File

@@ -646,12 +646,14 @@ class NeonEnvBuilder:
return env
# TODO WAN: make new thing
def enable_remote_storage(
self,
remote_storage_kind: RemoteStorageKind,
test_name: str,
force_enable: bool = True,
):
# TODO: more of these?
if remote_storage_kind == RemoteStorageKind.NOOP:
return
elif remote_storage_kind == RemoteStorageKind.LOCAL_FS:
@@ -673,13 +675,14 @@ class NeonEnvBuilder:
assert force_enable or self.remote_storage is None, "remote storage is enabled already"
self.remote_storage = LocalFsStorage(Path(self.repo_dir / "local_fs_remote_storage"))
# TODO: do we need a new method here? for remote_ext_config because we are not the pageserver?
# or is it ok to re-use the s3 bucket?
# TODO: do we need a new method here? for remote_ext_config because we are not the pageserver? A: YES
def enable_mock_s3_remote_storage(self, bucket_name: str, force_enable: bool = True):
"""
Sets up the pageserver to use the S3 mock server, creates the bucket, if it's not present already.
Starts up the mock server, if that does not run yet.
Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`.
TODO: this also does stuff for ext_remote_storage bucket
"""
assert force_enable or self.remote_storage is None, "remote storage is enabled already"
mock_endpoint = self.mock_s3_server.endpoint()
@@ -702,6 +705,16 @@ class NeonEnvBuilder:
secret_key=self.mock_s3_server.secret_key(),
)
ext_bucket_name = f"ext_{bucket_name}"
self.remote_storage_client.create_bucket(Bucket=ext_bucket_name)
self.ext_remote_storage = S3Storage(
bucket_name=ext_bucket_name,
endpoint=mock_endpoint,
bucket_region=mock_region,
access_key=self.mock_s3_server.access_key(),
secret_key=self.mock_s3_server.secret_key(),
)
def enable_real_s3_remote_storage(self, test_name: str, force_enable: bool = True):
"""
Sets up configuration to use real s3 endpoint without mock server
@@ -904,6 +917,8 @@ class NeonEnv:
self.neon_binpath = config.neon_binpath
self.pg_distrib_dir = config.pg_distrib_dir
self.endpoint_counter = 0
self.remote_storage_client = config.remote_storage_client
self.ext_remote_storage = config.ext_remote_storage
# generate initial tenant ID here instead of letting 'neon init' generate it,
# so that we don't need to dig it out of the config file afterwards.

19
test_runner/regress/test_download_extensions.py
View File

@@ -6,6 +6,8 @@ from fixtures.neon_fixtures import (
)
import json
TEST_EXT_PATH = "v15/share/extension/test_ext.control"
def test_file_download(neon_env_builder: NeonEnvBuilder):
"""
@@ -26,22 +28,17 @@ def test_file_download(neon_env_builder: NeonEnvBuilder):
neon_env_builder.num_safekeepers = 3
env = neon_env_builder.init_start()
TEST_EXT_PATH = "v15/share/extension/test_ext.control"
# TODO: we shouldn't be using neon_env_builder.remote_storage_client,
# we should pass the remote_storage_client to env in the builder.
# 4. Upload test_ext.control file to the bucket
# In the non-mock version this is done by CI/CD
with open("test_ext.control", "rb") as data:
neon_env_builder.remote_storage_client.upload_fileobj(
data, neon_env_builder.remote_storage.bucket_name, TEST_EXT_PATH
env.remote_storage_client.upload_fileobj(
data, env.ext_remote_storage.bucket_name, TEST_EXT_PATH
)
# 5. Download file from the bucket to correct local location
# Later this will be replaced by our rust code
resp = neon_env_builder.remote_storage_client.get_object(
Bucket=neon_env_builder.remote_storage.bucket_name, Key=TEST_EXT_PATH
resp = env.remote_storage_client.get_object(
Bucket=env.ext_remote_storage.bucket_name, Key=TEST_EXT_PATH
)
response = resp["Body"]
for pgres_version in ("v15", "v14"):
@@ -54,9 +51,9 @@ def test_file_download(neon_env_builder: NeonEnvBuilder):
remote_ext_config = json.dumps(
{
"bucket": neon_env_builder.remote_storage.bucket_name,
"bucket": env.ext_remote_storage.bucket_name,
"region": "us-east-1",
"endpoint": neon_env_builder.remote_storage.endpoint,
"endpoint": env.ext_remote_storage.endpoint,
}
)