rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-13 16:32:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add gc churn threshold knob (#1594)

Browse Source 
Signed-off-by: Dhammika Pathirana <dhammika@gmail.com>
This commit is contained in:
Dhammika Pathirana
2022-04-29 11:48:56 -07:00
parent 038ea4c128
commit f3f12db2cb
8 changed files with 45 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
control_plane/src/storage.rs
View File

@@ -369,6 +369,10 @@ impl PageServerNode {
.map(|x| x.parse::<u64>())
.transpose()?,
gc_period: settings.get("gc_period").map(|x| x.to_string()),
image_creation_threshold: settings
.get("image_creation_threshold")
.map(|x| x.parse::<usize>())
.transpose()?,
pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
})
.send()?
@@ -405,6 +409,9 @@ impl PageServerNode {
.get("gc_horizon")
.map(|x| x.parse::<u64>().unwrap()),
gc_period: settings.get("gc_period").map(|x| x.to_string()),
image_creation_threshold: settings
.get("image_creation_threshold")
.map(|x| x.parse::<usize>().unwrap()),
pitr_interval: settings.get("pitr_interval").map(|x| x.to_string()),
})
.send()?

1
pageserver/src/config.rs
View File

@@ -75,6 +75,7 @@ pub mod defaults {
#gc_period = '{DEFAULT_GC_PERIOD}'
#gc_horizon = {DEFAULT_GC_HORIZON}
#image_creation_threshold = {DEFAULT_IMAGE_CREATION_THRESHOLD}
#pitr_interval = '{DEFAULT_PITR_INTERVAL}'
# [remote_storage]

3
pageserver/src/http/models.rs
View File

@@ -31,6 +31,7 @@ pub struct TenantCreateRequest {
pub compaction_threshold: Option<usize>,
pub gc_horizon: Option<u64>,
pub gc_period: Option<String>,
pub image_creation_threshold: Option<usize>,
pub pitr_interval: Option<String>,
}
@@ -65,6 +66,7 @@ pub struct TenantConfigRequest {
pub compaction_threshold: Option<usize>,
pub gc_horizon: Option<u64>,
pub gc_period: Option<String>,
pub image_creation_threshold: Option<usize>,
pub pitr_interval: Option<String>,
}
@@ -78,6 +80,7 @@ impl TenantConfigRequest {
compaction_threshold: None,
gc_horizon: None,
gc_period: None,
image_creation_threshold: None,
pitr_interval: None,
}
}

2
pageserver/src/http/routes.rs
View File

@@ -387,6 +387,7 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
}
tenant_conf.gc_horizon = request_data.gc_horizon;
tenant_conf.image_creation_threshold = request_data.image_creation_threshold;
if let Some(pitr_interval) = request_data.pitr_interval {
tenant_conf.pitr_interval =
@@ -434,6 +435,7 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
Some(humantime::parse_duration(&gc_period).map_err(ApiError::from_err)?);
}
tenant_conf.gc_horizon = request_data.gc_horizon;
tenant_conf.image_creation_threshold = request_data.image_creation_threshold;
if let Some(pitr_interval) = request_data.pitr_interval {
tenant_conf.pitr_interval =

25
pageserver/src/layered_repository.rs
View File

@@ -528,6 +528,13 @@ impl LayeredRepository {
.unwrap_or(self.conf.default_tenant_conf.gc_period)
}
pub fn get_image_creation_threshold(&self) -> usize {
let tenant_conf = self.tenant_conf.read().unwrap();
tenant_conf
.image_creation_threshold
.unwrap_or(self.conf.default_tenant_conf.image_creation_threshold)
}
pub fn get_pitr_interval(&self) -> Duration {
let tenant_conf = self.tenant_conf.read().unwrap();
tenant_conf
@@ -1152,6 +1159,13 @@ impl LayeredTimeline {
.unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
}
fn get_image_creation_threshold(&self) -> usize {
let tenant_conf = self.tenant_conf.read().unwrap();
tenant_conf
.image_creation_threshold
.unwrap_or(self.conf.default_tenant_conf.image_creation_threshold)
}
/// Open a Timeline handle.
///
/// Loads the metadata for the timeline into memory, but not the layer map.
@@ -1821,7 +1835,7 @@ impl LayeredTimeline {
// 2. Create new image layers for partitions that have been modified
// "enough".
for part in partitioning.parts.iter() {
if self.time_for_new_image_layer(part, lsn, 3)? {
if self.time_for_new_image_layer(part, lsn)? {
self.create_image_layer(part, lsn)?;
}
}
@@ -1839,12 +1853,7 @@ impl LayeredTimeline {
}
// Is it time to create a new image layer for the given partition?
fn time_for_new_image_layer(
&self,
partition: &KeySpace,
lsn: Lsn,
threshold: usize,
) -> Result<bool> {
fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> Result<bool> {
let layers = self.layers.read().unwrap();
for part_range in &partition.ranges {
@@ -1862,7 +1871,7 @@ impl LayeredTimeline {
"range {}-{}, has {} deltas on this timeline",
img_range.start, img_range.end, num_deltas
);
if num_deltas >= threshold {
if num_deltas >= self.get_image_creation_threshold() {
return Ok(true);
}
}

2
pageserver/src/page_service.rs
View File

@@ -694,6 +694,7 @@ impl postgres_backend::Handler for PageServerHandler {
RowDescriptor::int8_col(b"compaction_threshold"),
RowDescriptor::int8_col(b"gc_horizon"),
RowDescriptor::int8_col(b"gc_period"),
RowDescriptor::int8_col(b"image_creation_threshold"),
RowDescriptor::int8_col(b"pitr_interval"),
]))?
.write_message_noflush(&BeMessage::DataRow(&[
@@ -708,6 +709,7 @@ impl postgres_backend::Handler for PageServerHandler {
Some(repo.get_compaction_threshold().to_string().as_bytes()),
Some(repo.get_gc_horizon().to_string().as_bytes()),
Some(repo.get_gc_period().as_secs().to_string().as_bytes()),
Some(repo.get_image_creation_threshold().to_string().as_bytes()),
Some(repo.get_pitr_interval().as_secs().to_string().as_bytes()),
]))?
.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;

1
pageserver/src/repository.rs
View File

@@ -467,6 +467,7 @@ pub mod repo_harness {
compaction_threshold: Some(tenant_conf.compaction_threshold),
gc_horizon: Some(tenant_conf.gc_horizon),
gc_period: Some(tenant_conf.gc_period),
image_creation_threshold: Some(tenant_conf.image_creation_threshold),
pitr_interval: Some(tenant_conf.pitr_interval),
}
}

12
pageserver/src/tenant_config.rs
View File

@@ -32,6 +32,7 @@ pub mod defaults {
pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
pub const DEFAULT_GC_PERIOD: &str = "100 s";
pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3;
pub const DEFAULT_PITR_INTERVAL: &str = "30 days";
}
@@ -59,6 +60,8 @@ pub struct TenantConf {
// Interval at which garbage collection is triggered.
#[serde(with = "humantime_serde")]
pub gc_period: Duration,
// Delta layer churn threshold to create L1 image layers.
pub image_creation_threshold: usize,
// Determines how much history is retained, to allow
// branching and read replicas at an older point in time.
// The unit is time.
@@ -79,6 +82,7 @@ pub struct TenantConfOpt {
pub gc_horizon: Option<u64>,
#[serde(with = "humantime_serde")]
pub gc_period: Option<Duration>,
pub image_creation_threshold: Option<usize>,
#[serde(with = "humantime_serde")]
pub pitr_interval: Option<Duration>,
}
@@ -100,6 +104,9 @@ impl TenantConfOpt {
.unwrap_or(global_conf.compaction_threshold),
gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
image_creation_threshold: self
.image_creation_threshold
.unwrap_or(global_conf.image_creation_threshold),
pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
}
}
@@ -123,6 +130,9 @@ impl TenantConfOpt {
if let Some(gc_period) = other.gc_period {
self.gc_period = Some(gc_period);
}
if let Some(image_creation_threshold) = other.image_creation_threshold {
self.image_creation_threshold = Some(image_creation_threshold);
}
if let Some(pitr_interval) = other.pitr_interval {
self.pitr_interval = Some(pitr_interval);
}
@@ -142,6 +152,7 @@ impl TenantConf {
gc_horizon: DEFAULT_GC_HORIZON,
gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
.expect("cannot parse default gc period"),
image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD,
pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL)
.expect("cannot parse default PITR interval"),
}
@@ -162,6 +173,7 @@ impl TenantConf {
compaction_threshold: defaults::DEFAULT_COMPACTION_THRESHOLD,
gc_horizon: defaults::DEFAULT_GC_HORIZON,
gc_period: Duration::from_secs(10),
image_creation_threshold: defaults::DEFAULT_IMAGE_CREATION_THRESHOLD,
pitr_interval: Duration::from_secs(60 * 60),
}
}