Fix tenant config parsing. Add a test

Cargo.lock

@@ -1073,6 +1073,16 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
+[[package]]
+name = "humantime-serde"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c"
+dependencies = [
+ "humantime",
+ "serde",
+]
+
 [[package]]
 name = "hyper"
 version = "0.14.17"
@@ -1626,6 +1636,7 @@ dependencies = [
  "hex",
  "hex-literal",
  "humantime",
+ "humantime-serde",
  "hyper",
  "itertools",
  "lazy_static",

pageserver/Cargo.toml

@@ -35,6 +35,7 @@ humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 serde_with = "1.12.0"
+humantime-serde = "1.1.1"
 
 pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
 

pageserver/src/config.rs

@@ -439,7 +439,7 @@ impl PageServerConf {
                 "remote_storage" => {
                     builder.remote_storage_config(Some(Self::parse_remote_storage_config(item)?))
                 }
-                "tenant_conf" => {
+                "tenant_config" => {
                     t_conf = Self::parse_toml_tenant_conf(item)?;
                 }
                 "id" => builder.id(ZNodeId(parse_toml_u64(key, item)?)),

pageserver/src/layered_repository.rs

@@ -690,7 +690,7 @@ impl LayeredRepository {
         let mut tenant_conf: TenantConfOpt = Default::default();
         for (key, item) in toml.iter() {
             match key {
-                "tenant_conf" => {
+                "tenant_config" => {
                     tenant_conf = PageServerConf::parse_toml_tenant_conf(item)?;
                 }
                 _ => bail!("unrecognized pageserver option '{}'", key),
@@ -712,7 +712,7 @@ impl LayeredRepository {
         let mut conf_content = r#"# This file contains a specific per-tenant's config.
 #  It is read in case of pageserver restart.
 
-# [tenant_config]
+[tenant_config]
 "#
         .to_string();
 

pageserver/src/tenant_config.rs

@@ -47,6 +47,7 @@ pub struct TenantConf {
     // This parameter determines L1 layer file size.
     pub compaction_target_size: u64,
     // How often to check if there's compaction work to be done.
+    #[serde(with = "humantime_serde")]
     pub compaction_period: Duration,
     // Level0 delta layer threshold for compaction.
     pub compaction_threshold: usize,
@@ -56,11 +57,13 @@ pub struct TenantConf {
     // Page versions older than this are garbage collected away.
     pub gc_horizon: u64,
     // Interval at which garbage collection is triggered.
+    #[serde(with = "humantime_serde")]
     pub gc_period: Duration,
     // Determines how much history is retained, to allow
     // branching and read replicas at an older point in time.
     // The unit is time.
     // Page versions older than this are garbage collected away.
+    #[serde(with = "humantime_serde")]
     pub pitr_interval: Duration,
 }
 
@@ -70,10 +73,13 @@ pub struct TenantConf {
 pub struct TenantConfOpt {
     pub checkpoint_distance: Option<u64>,
     pub compaction_target_size: Option<u64>,
+    #[serde(with = "humantime_serde")]
     pub compaction_period: Option<Duration>,
     pub compaction_threshold: Option<usize>,
     pub gc_horizon: Option<u64>,
+    #[serde(with = "humantime_serde")]
     pub gc_period: Option<Duration>,
+    #[serde(with = "humantime_serde")]
     pub pitr_interval: Option<Duration>,
 }
 

test_runner/batch_others/test_tenant_conf.py

@@ -3,21 +3,22 @@ from contextlib import closing
 import pytest
 
 from fixtures.zenith_fixtures import ZenithEnvBuilder
+from fixtures.log_helper import log
 
 
 def test_tenant_config(zenith_env_builder: ZenithEnvBuilder):
+    # set some non-default global config
+    zenith_env_builder.pageserver_config_override = '''
+page_cache_size=444;
+wait_lsn_timeout='111 s';
+tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}'''
+
     env = zenith_env_builder.init_start()
     """Test per tenant configuration"""
-    tenant = env.zenith_cli.create_tenant(
-        conf={
-            'checkpoint_distance': '10000',
-            'compaction_target_size': '1048576',
-            'compaction_period': '60sec',
-            'compaction_threshold': '20',
-            'gc_horizon': '1024',
-            'gc_period': '100sec',
-            'pitr_interval': '3600sec',
-        })
+    tenant = env.zenith_cli.create_tenant(conf={
+        'checkpoint_distance': '20000',
+        'gc_period': '30sec',
+    })
 
     env.zenith_cli.create_timeline(f'test_tenant_conf', tenant_id=tenant)
     pg = env.postgres.create_start(
@@ -26,24 +27,44 @@ def test_tenant_config(zenith_env_builder: ZenithEnvBuilder):
         tenant,
     )
 
+    # check the configuration of the default tenant
+    # it should match global configuration
+    with closing(env.pageserver.connect()) as psconn:
+        with psconn.cursor() as pscur:
+            pscur.execute(f"show {env.initial_tenant.hex}")
+            res = pscur.fetchone()
+            log.info(f"initial_tenant res: {res}")
+            assert res == (10000, 1048576, 1, 10, 67108864, 100, 2592000)
+
+    # check the configuration of the new tenant
     with closing(env.pageserver.connect()) as psconn:
         with psconn.cursor() as pscur:
             pscur.execute(f"show {tenant.hex}")
-            assert pscur.fetchone() == (10000, 1048576, 60, 20, 1024, 100, 3600)
+            res = pscur.fetchone()
+            log.info(f"res: {res}")
+            assert res == (20000, 1048576, 1, 10, 67108864, 30, 2592000)
 
     # update the config and ensure that it has changed
     env.zenith_cli.config_tenant(tenant_id=tenant,
                                  conf={
-                                     'checkpoint_distance': '100000',
-                                     'compaction_target_size': '1048576',
-                                     'compaction_period': '30sec',
-                                     'compaction_threshold': '15',
-                                     'gc_horizon': '256',
-                                     'gc_period': '10sec',
-                                     'pitr_interval': '360sec',
+                                     'checkpoint_distance': '15000',
+                                     'gc_period': '80sec',
                                  })
 
     with closing(env.pageserver.connect()) as psconn:
         with psconn.cursor() as pscur:
             pscur.execute(f"show {tenant.hex}")
-            assert pscur.fetchone() == (100000, 1048576, 30, 15, 256, 10, 360)
+            res = pscur.fetchone()
+            log.info(f"after config res: {res}")
+            assert res == (15000, 1048576, 1, 10, 67108864, 80, 2592000)
+
+    # restart the pageserver and ensure that the config is still correct
+    env.pageserver.stop()
+    env.pageserver.start()
+
+    with closing(env.pageserver.connect()) as psconn:
+        with psconn.cursor() as pscur:
+            pscur.execute(f"show {tenant.hex}")
+            res = pscur.fetchone()
+            log.info(f"after restart res: {res}")
+            assert res == (15000, 1048576, 1, 10, 67108864, 80, 2592000)