diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 7520ad9304..3a63bf6960 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -369,6 +369,10 @@ impl PageServerNode { .map(|x| x.parse::()) .transpose()?, gc_period: settings.get("gc_period").map(|x| x.to_string()), + image_creation_threshold: settings + .get("image_creation_threshold") + .map(|x| x.parse::()) + .transpose()?, pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()), }) .send()? @@ -405,6 +409,9 @@ impl PageServerNode { .get("gc_horizon") .map(|x| x.parse::().unwrap()), gc_period: settings.get("gc_period").map(|x| x.to_string()), + image_creation_threshold: settings + .get("image_creation_threshold") + .map(|x| x.parse::().unwrap()), pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()), }) .send()? diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index aed7eabb76..14ca976448 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -75,6 +75,7 @@ pub mod defaults { #gc_period = '{DEFAULT_GC_PERIOD}' #gc_horizon = {DEFAULT_GC_HORIZON} +#image_creation_threshold = {DEFAULT_IMAGE_CREATION_THRESHOLD} #pitr_interval = '{DEFAULT_PITR_INTERVAL}' # [remote_storage] diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index b24b3dc316..e9aaa72416 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -31,6 +31,7 @@ pub struct TenantCreateRequest { pub compaction_threshold: Option, pub gc_horizon: Option, pub gc_period: Option, + pub image_creation_threshold: Option, pub pitr_interval: Option, } @@ -65,6 +66,7 @@ pub struct TenantConfigRequest { pub compaction_threshold: Option, pub gc_horizon: Option, pub gc_period: Option, + pub image_creation_threshold: Option, pub pitr_interval: Option, } @@ -78,6 +80,7 @@ impl TenantConfigRequest { compaction_threshold: None, gc_horizon: None, gc_period: None, + image_creation_threshold: None, pitr_interval: None, } } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index c589813d69..5903dea372 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -387,6 +387,7 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result usize { + let tenant_conf = self.tenant_conf.read().unwrap(); + tenant_conf + .image_creation_threshold + .unwrap_or(self.conf.default_tenant_conf.image_creation_threshold) + } + pub fn get_pitr_interval(&self) -> Duration { let tenant_conf = self.tenant_conf.read().unwrap(); tenant_conf @@ -1152,6 +1159,13 @@ impl LayeredTimeline { .unwrap_or(self.conf.default_tenant_conf.compaction_threshold) } + fn get_image_creation_threshold(&self) -> usize { + let tenant_conf = self.tenant_conf.read().unwrap(); + tenant_conf + .image_creation_threshold + .unwrap_or(self.conf.default_tenant_conf.image_creation_threshold) + } + /// Open a Timeline handle. /// /// Loads the metadata for the timeline into memory, but not the layer map. @@ -1821,7 +1835,7 @@ impl LayeredTimeline { // 2. Create new image layers for partitions that have been modified // "enough". for part in partitioning.parts.iter() { - if self.time_for_new_image_layer(part, lsn, 3)? { + if self.time_for_new_image_layer(part, lsn)? { self.create_image_layer(part, lsn)?; } } @@ -1839,12 +1853,7 @@ impl LayeredTimeline { } // Is it time to create a new image layer for the given partition? - fn time_for_new_image_layer( - &self, - partition: &KeySpace, - lsn: Lsn, - threshold: usize, - ) -> Result { + fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> Result { let layers = self.layers.read().unwrap(); for part_range in &partition.ranges { @@ -1862,7 +1871,7 @@ impl LayeredTimeline { "range {}-{}, has {} deltas on this timeline", img_range.start, img_range.end, num_deltas ); - if num_deltas >= threshold { + if num_deltas >= self.get_image_creation_threshold() { return Ok(true); } } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index ec08a840b0..0adafab8ba 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -694,6 +694,7 @@ impl postgres_backend::Handler for PageServerHandler { RowDescriptor::int8_col(b"compaction_threshold"), RowDescriptor::int8_col(b"gc_horizon"), RowDescriptor::int8_col(b"gc_period"), + RowDescriptor::int8_col(b"image_creation_threshold"), RowDescriptor::int8_col(b"pitr_interval"), ]))? .write_message_noflush(&BeMessage::DataRow(&[ @@ -708,6 +709,7 @@ impl postgres_backend::Handler for PageServerHandler { Some(repo.get_compaction_threshold().to_string().as_bytes()), Some(repo.get_gc_horizon().to_string().as_bytes()), Some(repo.get_gc_period().as_secs().to_string().as_bytes()), + Some(repo.get_image_creation_threshold().to_string().as_bytes()), Some(repo.get_pitr_interval().as_secs().to_string().as_bytes()), ]))? .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 6c75f035ca..5044f2bfc5 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -467,6 +467,7 @@ pub mod repo_harness { compaction_threshold: Some(tenant_conf.compaction_threshold), gc_horizon: Some(tenant_conf.gc_horizon), gc_period: Some(tenant_conf.gc_period), + image_creation_threshold: Some(tenant_conf.image_creation_threshold), pitr_interval: Some(tenant_conf.pitr_interval), } } diff --git a/pageserver/src/tenant_config.rs b/pageserver/src/tenant_config.rs index a175f6abbe..9bf223e59e 100644 --- a/pageserver/src/tenant_config.rs +++ b/pageserver/src/tenant_config.rs @@ -32,6 +32,7 @@ pub mod defaults { pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024; pub const DEFAULT_GC_PERIOD: &str = "100 s"; + pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3; pub const DEFAULT_PITR_INTERVAL: &str = "30 days"; } @@ -59,6 +60,8 @@ pub struct TenantConf { // Interval at which garbage collection is triggered. #[serde(with = "humantime_serde")] pub gc_period: Duration, + // Delta layer churn threshold to create L1 image layers. + pub image_creation_threshold: usize, // Determines how much history is retained, to allow // branching and read replicas at an older point in time. // The unit is time. @@ -79,6 +82,7 @@ pub struct TenantConfOpt { pub gc_horizon: Option, #[serde(with = "humantime_serde")] pub gc_period: Option, + pub image_creation_threshold: Option, #[serde(with = "humantime_serde")] pub pitr_interval: Option, } @@ -100,6 +104,9 @@ impl TenantConfOpt { .unwrap_or(global_conf.compaction_threshold), gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon), gc_period: self.gc_period.unwrap_or(global_conf.gc_period), + image_creation_threshold: self + .image_creation_threshold + .unwrap_or(global_conf.image_creation_threshold), pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval), } } @@ -123,6 +130,9 @@ impl TenantConfOpt { if let Some(gc_period) = other.gc_period { self.gc_period = Some(gc_period); } + if let Some(image_creation_threshold) = other.image_creation_threshold { + self.image_creation_threshold = Some(image_creation_threshold); + } if let Some(pitr_interval) = other.pitr_interval { self.pitr_interval = Some(pitr_interval); } @@ -142,6 +152,7 @@ impl TenantConf { gc_horizon: DEFAULT_GC_HORIZON, gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD) .expect("cannot parse default gc period"), + image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD, pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL) .expect("cannot parse default PITR interval"), } @@ -162,6 +173,7 @@ impl TenantConf { compaction_threshold: defaults::DEFAULT_COMPACTION_THRESHOLD, gc_horizon: defaults::DEFAULT_GC_HORIZON, gc_period: Duration::from_secs(10), + image_creation_threshold: defaults::DEFAULT_IMAGE_CREATION_THRESHOLD, pitr_interval: Duration::from_secs(60 * 60), } }