From 38c6f6ce1624e6ece59a3e3c83c59c0005dc2166 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 18 Jan 2022 11:38:51 +0200 Subject: [PATCH] Allow specifying custom endpoint in s3 --- pageserver/src/config.rs | 24 +++++++++++++++++++----- pageserver/src/remote_storage/rust_s3.rs | 14 ++++++++++---- test_runner/fixtures/zenith_fixtures.py | 14 ++++++++++---- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index e89ec562ed..3d64d830b5 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -144,6 +144,13 @@ pub struct S3Config { pub access_key_id: Option, /// "Password" to use when connecting to bucket. pub secret_access_key: Option, + /// A base URL to send S3 requests to. + /// By default, the endpoint is derived from a region name, assuming it's + /// an AWS S3 region name, erroring on wrong region name. + /// Endpoint provides a way to support other S3 flavors and their regions. + /// + /// Example: `http://127.0.0.1:5000` + pub endpoint: Option, } impl std::fmt::Debug for S3Config { @@ -349,6 +356,10 @@ impl PageServerConf { .get("prefix_in_bucket") .map(|prefix_in_bucket| parse_toml_string("prefix_in_bucket", prefix_in_bucket)) .transpose()?, + endpoint: toml + .get("endpoint") + .map(|endpoint| parse_toml_string("endpoint", endpoint)) + .transpose()?, }), (Some(local_path), None, None) => RemoteStorageKind::LocalFs(PathBuf::from( parse_toml_string("local_path", local_path)?, @@ -597,6 +608,7 @@ pg_distrib_dir='{}' let prefix_in_bucket = "test_prefix".to_string(); let access_key_id = "SOMEKEYAAAAASADSAH*#".to_string(); let secret_access_key = "SOMEsEcReTsd292v".to_string(); + let endpoint = "http://localhost:5000".to_string(); let max_concurrent_sync = NonZeroUsize::new(111).unwrap(); let max_sync_errors = NonZeroU32::new(222).unwrap(); @@ -609,12 +621,13 @@ bucket_name = '{}' bucket_region = '{}' prefix_in_bucket = '{}' access_key_id = '{}' -secret_access_key = '{}'"#, - max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key +secret_access_key = '{}' +endpoint = '{}'"#, + max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint ), format!( - "remote_storage={{max_concurrent_sync={}, max_sync_errors={}, bucket_name='{}', bucket_region='{}', prefix_in_bucket='{}', access_key_id='{}', secret_access_key='{}'}}", - max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key + "remote_storage={{max_concurrent_sync={}, max_sync_errors={}, bucket_name='{}', bucket_region='{}', prefix_in_bucket='{}', access_key_id='{}', secret_access_key='{}', endpoint='{}'}}", + max_concurrent_sync, max_sync_errors, bucket_name, bucket_region, prefix_in_bucket, access_key_id, secret_access_key, endpoint ), ]; @@ -648,7 +661,8 @@ pg_distrib_dir='{}' bucket_region: bucket_region.clone(), access_key_id: Some(access_key_id.clone()), secret_access_key: Some(secret_access_key.clone()), - prefix_in_bucket: Some(prefix_in_bucket.clone()) + prefix_in_bucket: Some(prefix_in_bucket.clone()), + endpoint: Some(endpoint.clone()) }), }, "Remote storage config should correctly parse the S3 config" diff --git a/pageserver/src/remote_storage/rust_s3.rs b/pageserver/src/remote_storage/rust_s3.rs index 0ef181d9b5..757ff8ffe9 100644 --- a/pageserver/src/remote_storage/rust_s3.rs +++ b/pageserver/src/remote_storage/rust_s3.rs @@ -58,10 +58,16 @@ pub struct S3 { impl S3 { /// Creates the storage, errors if incorrect AWS S3 configuration provided. pub fn new(aws_config: &S3Config, pageserver_workdir: &'static Path) -> anyhow::Result { - let region = aws_config - .bucket_region - .parse::() - .context("Failed to parse the s3 region from config")?; + let region = match aws_config.endpoint.clone() { + Some(endpoint) => Region::Custom { + endpoint, + region: aws_config.bucket_region.clone(), + }, + None => aws_config + .bucket_region + .parse::() + .context("Failed to parse the s3 region from config")?, + }; let credentials = Credentials::new( aws_config.access_key_id.as_deref(), aws_config.secret_access_key.as_deref(), diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index d2c5973bfc..0c8292a166 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -717,8 +717,9 @@ class LocalFsStorage: class S3Storage: bucket: str region: str - access_key: str - secret_key: str + access_key: Optional[str] + secret_key: Optional[str] + endpoint: Optional[str] RemoteStorage = Union[LocalFsStorage, S3Storage] @@ -791,8 +792,13 @@ def append_pageserver_param_overrides(params_to_update: List[str], pageserver_storage_override = f"local_path='{pageserver_remote_storage.root}'" elif isinstance(pageserver_remote_storage, S3Storage): pageserver_storage_override = f"bucket_name='{pageserver_remote_storage.bucket}',\ - bucket_region='{pageserver_remote_storage.region}',access_key_id='{pageserver_remote_storage.access_key}',\ - secret_access_key='{pageserver_remote_storage.secret_key}'" + bucket_region='{pageserver_remote_storage.region}'" + if pageserver_remote_storage.access_key is not None: + pageserver_storage_override += f",access_key_id='{pageserver_remote_storage.access_key}'" + if pageserver_remote_storage.secret_key is not None: + pageserver_storage_override += f",secret_access_key='{pageserver_remote_storage.secret_key}'" + if pageserver_remote_storage.endpoint is not None: + pageserver_storage_override += f",endpoint='{pageserver_remote_storage.endpoint}'" else: raise Exception(f'Unknown storage configuration {pageserver_remote_storage}')