rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-16 09:52:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add compress_image_layer property to TenantConfig

Browse Source
This commit is contained in:
Konstantin Knizhnik
2024-03-11 22:13:11 +02:00
parent 0df6c41eaa
commit 8c7136b057
8 changed files with 63 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
control_plane/src/pageserver.rs
View File

@@ -384,6 +384,11 @@ impl PageServerNode {
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'trace_read_requests' as bool")?,
compress_image_layer: settings
.remove("compress_image_layer")
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'compress_image_layer' as bool")?,
eviction_policy: settings
.remove("eviction_policy")
.map(serde_json::from_str)
@@ -496,6 +501,11 @@ impl PageServerNode {
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'trace_read_requests' as bool")?,
compress_image_layer: settings
.remove("compress_image_layer")
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'compress_image_layer' as bool")?,
eviction_policy: settings
.remove("eviction_policy")
.map(serde_json::from_str)

1
libs/pageserver_api/src/models.rs
View File

@@ -294,6 +294,7 @@ pub struct TenantConfig {
pub lagging_wal_timeout: Option<String>,
pub max_lsn_wal_lag: Option<NonZeroU64>,
pub trace_read_requests: Option<bool>,
pub compress_image_layer: Option<bool>,
pub eviction_policy: Option<EvictionPolicy>,
pub min_resident_size_override: Option<u64>,
pub evictions_low_residence_duration_metric_threshold: Option<String>,

8
pageserver/src/config.rs
View File

@@ -1538,6 +1538,7 @@ broker_endpoint = '{broker_endpoint}'
let broker_endpoint = "http://127.0.0.1:7777";
let trace_read_requests = true;
let compress_image_layer = true;
let config_string = format!(
r#"{ALL_BASE_VALUES_TOML}
@@ -1545,7 +1546,8 @@ pg_distrib_dir='{pg_distrib_dir}'
broker_endpoint = '{broker_endpoint}'
[tenant_config]
trace_read_requests = {trace_read_requests}"#,
trace_read_requests = {trace_read_requests},
compress_image_layer = {compress_image_layer}"#,
);
let toml = config_string.parse()?;
@@ -1555,6 +1557,10 @@ trace_read_requests = {trace_read_requests}"#,
conf.default_tenant_conf.trace_read_requests, trace_read_requests,
"Tenant config from pageserver config file should be parsed and udpated values used as defaults for all tenants",
);
assert_eq!(
conf.default_tenant_conf.compress_image_layer, compress_image_layer,
"Tenant config from pageserver config file should be parsed and udpated values used as defaults for all tenants",
);
Ok(())
}

8
pageserver/src/tenant.rs
View File

@@ -2288,6 +2288,13 @@ impl Tenant {
.unwrap_or(self.conf.default_tenant_conf.trace_read_requests)
}
pub fn get_compress_image_layer(&self) -> bool {
let tenant_conf = self.tenant_conf.read().unwrap().tenant_conf.clone();
tenant_conf
.compress_image_layer
.unwrap_or(self.conf.default_tenant_conf.compress_image_layer)
}
pub fn get_min_resident_size_override(&self) -> Option<u64> {
let tenant_conf = self.tenant_conf.read().unwrap().tenant_conf.clone();
tenant_conf
@@ -3637,6 +3644,7 @@ pub(crate) mod harness {
lagging_wal_timeout: Some(tenant_conf.lagging_wal_timeout),
max_lsn_wal_lag: Some(tenant_conf.max_lsn_wal_lag),
trace_read_requests: Some(tenant_conf.trace_read_requests),
compress_image_layer: Some(tenant_conf.compress_image_layer),
eviction_policy: Some(tenant_conf.eviction_policy),
min_resident_size_override: tenant_conf.min_resident_size_override,
evictions_low_residence_duration_metric_threshold: Some(

10
pageserver/src/tenant/config.rs
View File

@@ -345,6 +345,7 @@ pub struct TenantConf {
/// to avoid eager reconnects.
pub max_lsn_wal_lag: NonZeroU64,
pub trace_read_requests: bool,
pub compress_image_layer: bool,
pub eviction_policy: EvictionPolicy,
pub min_resident_size_override: Option<u64>,
// See the corresponding metric's help string.
@@ -429,6 +430,10 @@ pub struct TenantConfOpt {
#[serde(default)]
pub trace_read_requests: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub compress_image_layer: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
pub eviction_policy: Option<EvictionPolicy>,
@@ -492,6 +497,9 @@ impl TenantConfOpt {
trace_read_requests: self
.trace_read_requests
.unwrap_or(global_conf.trace_read_requests),
compress_image_layer: self
.compress_image_layer
.unwrap_or(global_conf.compress_image_layer),
eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
min_resident_size_override: self
.min_resident_size_override
@@ -538,6 +546,7 @@ impl Default for TenantConf {
max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
.expect("cannot parse default max walreceiver Lsn wal lag"),
trace_read_requests: false,
compress_image_layer: false,
eviction_policy: EvictionPolicy::NoEviction,
min_resident_size_override: None,
evictions_low_residence_duration_metric_threshold: humantime::parse_duration(
@@ -612,6 +621,7 @@ impl From<TenantConfOpt> for models::TenantConfig {
lagging_wal_timeout: value.lagging_wal_timeout.map(humantime),
max_lsn_wal_lag: value.max_lsn_wal_lag,
trace_read_requests: value.trace_read_requests,
compress_image_layer: value.compress_image_layer,
eviction_policy: value.eviction_policy,
min_resident_size_override: value.min_resident_size_override,
evictions_low_residence_duration_metric_threshold: value

30
pageserver/src/tenant/storage_layer/image_layer.rs
View File

@@ -598,6 +598,7 @@ struct ImageLayerWriterInner {
timeline_id: TimelineId,
tenant_shard_id: TenantShardId,
key_range: Range<Key>,
compression: bool,
lsn: Lsn,
blob_writer: BlobWriter<false>,
@@ -609,16 +610,17 @@ impl ImageLayerWriterInner {
/// Start building a new image layer.
///
async fn new(
conf: &'static PageServerConf,
timeline_id: TimelineId,
tenant_shard_id: TenantShardId,
timeline: &Arc<Timeline>,
key_range: &Range<Key>,
lsn: Lsn,
) -> anyhow::Result<Self> {
let timeline_id = timeline.timeline_id;
let tenant_shard_id = timeline.tenant_shard_id;
let compression = timeline.get_image_layer_compression();
// Create the file initially with a temporary filename.
// We'll atomically rename it to the final name when we're done.
let path = ImageLayer::temp_path_for(
conf,
timeline.conf,
timeline_id,
tenant_shard_id,
&ImageFileName {
@@ -645,11 +647,12 @@ impl ImageLayerWriterInner {
let tree_builder = DiskBtreeBuilder::new(block_buf);
let writer = Self {
conf,
conf: timeline.conf,
path,
timeline_id,
tenant_shard_id,
key_range: key_range.clone(),
compression,
lsn,
tree: tree_builder,
blob_writer,
@@ -665,7 +668,13 @@ impl ImageLayerWriterInner {
///
async fn put_image(&mut self, key: Key, img: Bytes) -> anyhow::Result<()> {
ensure!(self.key_range.contains(&key));
let off = self.blob_writer.write_compressed_blob(img).await?;
let off = if self.compression {
self.blob_writer.write_compressed_blob(img).await?
} else {
let (_img, res) = self.blob_writer.write_blob(img).await;
// TODO: re-use the buffer for `img` further upstack
res?
};
let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
key.write_to_byte_slice(&mut keybuf);
self.tree.append(&keybuf, off)?;
@@ -770,17 +779,12 @@ impl ImageLayerWriter {
/// Start building a new image layer.
///
pub async fn new(
conf: &'static PageServerConf,
timeline_id: TimelineId,
tenant_shard_id: TenantShardId,
timeline: &Arc<Timeline>,
key_range: &Range<Key>,
lsn: Lsn,
) -> anyhow::Result<ImageLayerWriter> {
Ok(Self {
inner: Some(
ImageLayerWriterInner::new(conf, timeline_id, tenant_shard_id, key_range, lsn)
.await?,
),
inner: Some(ImageLayerWriterInner::new(timeline, key_range, lsn).await?),
})
}

18
pageserver/src/tenant/timeline.rs
View File

@@ -182,7 +182,7 @@ pub(crate) struct AuxFilesState {
}
pub struct Timeline {
conf: &'static PageServerConf,
pub(crate) conf: &'static PageServerConf,
tenant_conf: Arc<RwLock<AttachedTenantConf>>,
myself: Weak<Self>,
@@ -1515,6 +1515,13 @@ impl Timeline {
.unwrap_or(default_tenant_conf.evictions_low_residence_duration_metric_threshold)
}
pub fn get_image_layer_compression(&self) -> bool {
let tenant_conf = self.tenant_conf.read().unwrap().tenant_conf.clone();
tenant_conf
.compress_image_layer
.unwrap_or(self.conf.default_tenant_conf.compress_image_layer)
}
pub(super) fn tenant_conf_updated(&self) {
// NB: Most tenant conf options are read by background loops, so,
// changes will automatically be picked up.
@@ -3458,14 +3465,7 @@ impl Timeline {
continue;
}
let mut image_layer_writer = ImageLayerWriter::new(
self.conf,
self.timeline_id,
self.tenant_shard_id,
&img_range,
lsn,
)
.await?;
let mut image_layer_writer = ImageLayerWriter::new(self, &img_range, lsn).await?;
fail_point!("image-layer-writer-fail-before-finish", |_| {
Err(CreateImageLayersError::Other(anyhow::anyhow!(

9
pageserver/src/tenant/timeline/compaction.rs
View File

@@ -997,14 +997,7 @@ impl TimelineAdaptor {
) -> Result<(), PageReconstructError> {
let timer = self.timeline.metrics.create_images_time_histo.start_timer();
let mut image_layer_writer = ImageLayerWriter::new(
self.timeline.conf,
self.timeline.timeline_id,
self.timeline.tenant_shard_id,
key_range,
lsn,
)
.await?;
let mut image_layer_writer = ImageLayerWriter::new(&self.timeline, key_range, lsn).await?;
fail_point!("image-layer-writer-fail-before-finish", |_| {
Err(PageReconstructError::Other(anyhow::anyhow!(