From 71607877567ec69722a222197903fee4a3f0a678 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 7 Apr 2025 20:19:55 +0200 Subject: [PATCH] pageserver: add changed_bytes_from_parent consumption metric --- pageserver/src/consumption_metrics/metrics.rs | 40 +++++++++++++++++++ .../src/consumption_metrics/metrics/tests.rs | 14 +++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 71910011ea..a924d38ea5 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -27,6 +27,9 @@ pub(super) enum Name { /// Timeline logical size #[serde(rename = "timeline_logical_size")] LogicalSize, + /// Timeline delta from parent + #[serde(rename = "timeline_changed_bytes_from_parent")] + ChangesFromParent, /// Tenant remote size #[serde(rename = "remote_storage_size")] RemoteSize, @@ -175,6 +178,18 @@ impl MetricsKey { .absolute_values() } + const fn timeline_changed_bytes_from_parent( + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> AbsoluteValueFactory { + MetricsKey { + tenant_id, + timeline_id: Some(timeline_id), + metric: Name::ChangesFromParent, + } + .absolute_values() + } + /// [`Tenant::remote_size`] /// /// [`Tenant::remote_size`]: crate::tenant::Tenant::remote_size @@ -364,6 +379,7 @@ struct TimelineSnapshot { loaded_at: (Lsn, SystemTime), last_record_lsn: Lsn, current_exact_logical_size: Option, + changed_bytes_from_parent: Option, } impl TimelineSnapshot { @@ -399,10 +415,23 @@ impl TimelineSnapshot { } }; + // This is an approximation of how much data has changed on this branch vs. its ancestor: the + // number of bytes written to the WAL, clamped to the size of the branch. + let changed_bytes_from_parent = current_exact_logical_size.and_then(|size| { + if t.get_ancestor_lsn() == Lsn::MAX { + None + } else { + t.get_last_record_lsn() + .checked_sub(t.get_ancestor_lsn()) + .map(|wal_bytes| std::cmp::min(wal_bytes.0, size)) + } + }); + Ok(Some(TimelineSnapshot { loaded_at, last_record_lsn, current_exact_logical_size, + changed_bytes_from_parent, })) } } @@ -480,6 +509,17 @@ impl TimelineSnapshot { metrics.push(factory.at(now, size)); } } + + { + let factory = MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id); + let current_or_previous = self + .changed_bytes_from_parent + .or_else(|| cache.get(factory.key()).map(|item| item.value)); + + if let Some(size) = current_or_previous { + metrics.push(factory.at(now, size)); + } + } } } diff --git a/pageserver/src/consumption_metrics/metrics/tests.rs b/pageserver/src/consumption_metrics/metrics/tests.rs index 52b4fb8680..ea5f8648f4 100644 --- a/pageserver/src/consumption_metrics/metrics/tests.rs +++ b/pageserver/src/consumption_metrics/metrics/tests.rs @@ -18,6 +18,7 @@ fn startup_collected_timeline_metrics_before_advancing() { loaded_at: (disk_consistent_lsn, SystemTime::now()), last_record_lsn: disk_consistent_lsn, current_exact_logical_size: Some(0x42000), + changed_bytes_from_parent: Some(0x1000), }; let now = DateTime::::from(SystemTime::now()); @@ -33,7 +34,8 @@ fn startup_collected_timeline_metrics_before_advancing() { 0 ), MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0), - MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000) + MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000), + MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000) ] ); } @@ -60,6 +62,7 @@ fn startup_collected_timeline_metrics_second_round() { loaded_at: (disk_consistent_lsn, init), last_record_lsn: disk_consistent_lsn, current_exact_logical_size: Some(0x42000), + changed_bytes_from_parent: Some(0x1000), }; snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache); @@ -69,7 +72,8 @@ fn startup_collected_timeline_metrics_second_round() { &[ MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0), MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0), - MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000) + MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000), + MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000) ] ); } @@ -104,6 +108,7 @@ fn startup_collected_timeline_metrics_nth_round_at_same_lsn() { loaded_at: (disk_consistent_lsn, init), last_record_lsn: disk_consistent_lsn, current_exact_logical_size: Some(0x42000), + changed_bytes_from_parent: Some(0x1000), }; snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache); @@ -113,7 +118,8 @@ fn startup_collected_timeline_metrics_nth_round_at_same_lsn() { &[ MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(just_before, now, 0), MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0), - MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000) + MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000), + MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000) ] ); } @@ -141,6 +147,7 @@ fn post_restart_written_sizes_with_rolled_back_last_record_lsn() { loaded_at: (Lsn(50), at_restart), last_record_lsn: Lsn(50), current_exact_logical_size: None, + changed_bytes_from_parent: None, }; let mut cache = HashMap::from([ @@ -202,6 +209,7 @@ fn post_restart_current_exact_logical_size_uses_cached() { loaded_at: (Lsn(50), at_restart), last_record_lsn: Lsn(50), current_exact_logical_size: None, + changed_bytes_from_parent: Some(0x1000), }; let cache = HashMap::from([MetricsKey::timeline_logical_size(tenant_id, timeline_id)