diff --git a/Cargo.lock b/Cargo.lock index bac5dfb674..58125ca41c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1073,6 +1073,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hyper" version = "0.14.17" @@ -1626,6 +1636,7 @@ dependencies = [ "hex", "hex-literal", "humantime", + "humantime-serde", "hyper", "itertools", "lazy_static", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 6648d8417a..5607baf698 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -35,6 +35,7 @@ humantime = "2.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "1.12.0" +humantime-serde = "1.1.1" pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 8bfe8b57ec..aed7eabb76 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -439,7 +439,7 @@ impl PageServerConf { "remote_storage" => { builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?)) } - "tenant_conf" => { + "tenant_config" => { t_conf = Self::parse_toml_tenant_conf(item)?; } "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)), diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 679daa8248..d9e1244f2e 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -690,7 +690,7 @@ impl LayeredRepository { let mut tenant_conf: TenantConfOpt = Default::default(); for (key, item) in toml.iter() { match key { - "tenant_conf" => { + "tenant_config" => { tenant_conf = PageServerConf::parse_toml_tenant_conf(item)?; } _ => bail!("unrecognized pageserver option '{}'", key), @@ -712,7 +712,7 @@ impl LayeredRepository { let mut conf_content = r#"# This file contains a specific per-tenant's config. # It is read in case of pageserver restart. -# [tenant_config] +[tenant_config] "# .to_string(); diff --git a/pageserver/src/tenant_config.rs b/pageserver/src/tenant_config.rs index 818b6de1b1..a175f6abbe 100644 --- a/pageserver/src/tenant_config.rs +++ b/pageserver/src/tenant_config.rs @@ -47,6 +47,7 @@ pub struct TenantConf { // This parameter determines L1 layer file size. pub compaction_target_size: u64, // How often to check if there's compaction work to be done. + #[serde(with = "humantime_serde")] pub compaction_period: Duration, // Level0 delta layer threshold for compaction. pub compaction_threshold: usize, @@ -56,11 +57,13 @@ pub struct TenantConf { // Page versions older than this are garbage collected away. pub gc_horizon: u64, // Interval at which garbage collection is triggered. + #[serde(with = "humantime_serde")] pub gc_period: Duration, // Determines how much history is retained, to allow // branching and read replicas at an older point in time. // The unit is time. // Page versions older than this are garbage collected away. + #[serde(with = "humantime_serde")] pub pitr_interval: Duration, } @@ -70,10 +73,13 @@ pub struct TenantConf { pub struct TenantConfOpt { pub checkpoint_distance: Option, pub compaction_target_size: Option, + #[serde(with = "humantime_serde")] pub compaction_period: Option, pub compaction_threshold: Option, pub gc_horizon: Option, + #[serde(with = "humantime_serde")] pub gc_period: Option, + #[serde(with = "humantime_serde")] pub pitr_interval: Option, } diff --git a/test_runner/batch_others/test_tenant_conf.py b/test_runner/batch_others/test_tenant_conf.py index f74e6aad1d..64359a1dc3 100644 --- a/test_runner/batch_others/test_tenant_conf.py +++ b/test_runner/batch_others/test_tenant_conf.py @@ -3,21 +3,22 @@ from contextlib import closing import pytest from fixtures.zenith_fixtures import ZenithEnvBuilder +from fixtures.log_helper import log def test_tenant_config(zenith_env_builder: ZenithEnvBuilder): + # set some non-default global config + zenith_env_builder.pageserver_config_override = ''' +page_cache_size=444; +wait_lsn_timeout='111 s'; +tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}''' + env = zenith_env_builder.init_start() """Test per tenant configuration""" - tenant = env.zenith_cli.create_tenant( - conf={ - 'checkpoint_distance': '10000', - 'compaction_target_size': '1048576', - 'compaction_period': '60sec', - 'compaction_threshold': '20', - 'gc_horizon': '1024', - 'gc_period': '100sec', - 'pitr_interval': '3600sec', - }) + tenant = env.zenith_cli.create_tenant(conf={ + 'checkpoint_distance': '20000', + 'gc_period': '30sec', + }) env.zenith_cli.create_timeline(f'test_tenant_conf', tenant_id=tenant) pg = env.postgres.create_start( @@ -26,24 +27,44 @@ def test_tenant_config(zenith_env_builder: ZenithEnvBuilder): tenant, ) + # check the configuration of the default tenant + # it should match global configuration + with closing(env.pageserver.connect()) as psconn: + with psconn.cursor() as pscur: + pscur.execute(f"show {env.initial_tenant.hex}") + res = pscur.fetchone() + log.info(f"initial_tenant res: {res}") + assert res == (10000, 1048576, 1, 10, 67108864, 100, 2592000) + + # check the configuration of the new tenant with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: pscur.execute(f"show {tenant.hex}") - assert pscur.fetchone() == (10000, 1048576, 60, 20, 1024, 100, 3600) + res = pscur.fetchone() + log.info(f"res: {res}") + assert res == (20000, 1048576, 1, 10, 67108864, 30, 2592000) # update the config and ensure that it has changed env.zenith_cli.config_tenant(tenant_id=tenant, conf={ - 'checkpoint_distance': '100000', - 'compaction_target_size': '1048576', - 'compaction_period': '30sec', - 'compaction_threshold': '15', - 'gc_horizon': '256', - 'gc_period': '10sec', - 'pitr_interval': '360sec', + 'checkpoint_distance': '15000', + 'gc_period': '80sec', }) with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: pscur.execute(f"show {tenant.hex}") - assert pscur.fetchone() == (100000, 1048576, 30, 15, 256, 10, 360) + res = pscur.fetchone() + log.info(f"after config res: {res}") + assert res == (15000, 1048576, 1, 10, 67108864, 80, 2592000) + + # restart the pageserver and ensure that the config is still correct + env.pageserver.stop() + env.pageserver.start() + + with closing(env.pageserver.connect()) as psconn: + with psconn.cursor() as pscur: + pscur.execute(f"show {tenant.hex}") + res = pscur.fetchone() + log.info(f"after restart res: {res}") + assert res == (15000, 1048576, 1, 10, 67108864, 80, 2592000)