diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 114d725957..5e52fac064 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -253,12 +253,20 @@ pub struct LayerAccessStatFullDetails { pub access_kind: LayerAccessKind, } +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +#[serde(tag = "kind")] +pub enum LayerResidenceStatus { + Resident { timestamp_millis_since_epoch: u128 }, + Evicted { timestamp_millis_since_epoch: u128 }, +} + #[derive(Debug, Clone, Serialize)] pub struct LayerAccessStats { pub access_count_by_access_kind: HashMap, pub task_kind_access_flag: Vec<&'static str>, pub first: Option, pub most_recent: Vec, + pub most_recent_residence_changes: Vec, } #[derive(Debug, Copy, Clone, Serialize, Deserialize)] diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index e2670886e3..66cef4618c 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -88,21 +88,29 @@ pub enum ValueReconstructResult { Missing, } -#[derive(Default)] +#[derive(Default, Debug)] pub struct LayerAccessStats(Mutex); -#[derive(Clone)] +#[derive(Debug, Clone)] struct LayerAccessStatFullDetails { when: SystemTime, task_kind: TaskKind, access_kind: LayerAccessKind, } +#[derive(Debug, Clone)] struct LayerAccessStatsInner { first_access: Option, count_by_access_kind: EnumMap, task_kind_flag: EnumSet, last_accesses: VecDeque, + last_residence_changes: VecDeque, +} + +#[derive(Debug, Clone)] +pub enum LayerResidenceStatus { + Resident { timestamp: SystemTime }, + Evicted { timestamp: SystemTime }, } #[derive(Clone, Copy, strum_macros::EnumString)] @@ -111,6 +119,12 @@ pub enum LayerAccessStatsReset { AllStats, } +fn system_time_to_millis_since_epoch(ts: &SystemTime) -> u128 { + ts.duration_since(UNIX_EPOCH) + .expect("better to die in this unlikely case than report false stats") + .as_millis() +} + impl LayerAccessStatFullDetails { fn to_api_model(&self) -> pageserver_api::models::LayerAccessStatFullDetails { let Self { @@ -119,16 +133,42 @@ impl LayerAccessStatFullDetails { access_kind, } = self; pageserver_api::models::LayerAccessStatFullDetails { - when_millis_since_epoch: when - .duration_since(UNIX_EPOCH) - .expect("better to die in this unlikely case than report false stats") - .as_millis(), + when_millis_since_epoch: system_time_to_millis_since_epoch(when), task_kind: task_kind.into(), // into static str, powered by strum_macros access_kind: *access_kind, } } } +impl LayerResidenceStatus { + pub fn evicted() -> Self { + LayerResidenceStatus::Evicted { + timestamp: SystemTime::now(), + } + } + + pub fn resident() -> Self { + LayerResidenceStatus::Resident { + timestamp: SystemTime::now(), + } + } + + fn to_api_model(&self) -> pageserver_api::models::LayerResidenceStatus { + match self { + LayerResidenceStatus::Resident { timestamp } => { + pageserver_api::models::LayerResidenceStatus::Resident { + timestamp_millis_since_epoch: system_time_to_millis_since_epoch(timestamp), + } + } + LayerResidenceStatus::Evicted { timestamp } => { + pageserver_api::models::LayerResidenceStatus::Evicted { + timestamp_millis_since_epoch: system_time_to_millis_since_epoch(timestamp), + } + } + } + } +} + pub static LAYER_ACCESS_STATS_KILLSWITCH: AtomicBool = AtomicBool::new(false); impl Default for LayerAccessStatsInner { @@ -138,12 +178,36 @@ impl Default for LayerAccessStatsInner { count_by_access_kind: EnumMap::default(), task_kind_flag: EnumSet::default(), last_accesses: VecDeque::with_capacity(LayerAccessStats::LAST_ACCESSES_MAX_LEN), + last_residence_changes: VecDeque::with_capacity( + LayerAccessStats::LAST_RESIDENCE_CHANGES_MAX_LEN, + ), } } } impl LayerAccessStats { const LAST_ACCESSES_MAX_LEN: usize = 16; + const LAST_RESIDENCE_CHANGES_MAX_LEN: usize = 16; + + /// Creates a clone of `self` and records `new_status` in the clone. + /// The `new_status` is not recorded in `self` + pub(crate) fn clone_for_residence_change( + &self, + new_status: LayerResidenceStatus, + ) -> LayerAccessStats { + let mut clone = { + let inner = self.0.lock().unwrap(); + inner.clone() + }; + + // make room first to avoid reallocs + while clone.last_residence_changes.len() >= Self::LAST_RESIDENCE_CHANGES_MAX_LEN { + clone.last_residence_changes.pop_back(); + } + clone.last_residence_changes.push_front(new_status.clone()); + + LayerAccessStats(Mutex::new(clone)) + } fn record_access(&self, access_kind: LayerAccessKind, task_kind: TaskKind) { if LAYER_ACCESS_STATS_KILLSWITCH.load(atomic::Ordering::SeqCst) { @@ -184,6 +248,7 @@ impl LayerAccessStats { count_by_access_kind, task_kind_flag, last_accesses, + last_residence_changes, } = &*inner; pageserver_api::models::LayerAccessStats { access_count_by_access_kind: count_by_access_kind @@ -196,6 +261,10 @@ impl LayerAccessStats { .collect(), first: first_access.as_ref().map(|a| a.to_api_model()), most_recent: last_accesses.iter().map(|a| a.to_api_model()).collect(), + most_recent_residence_changes: last_residence_changes + .iter() + .map(|s| s.to_api_model()) + .collect(), } } } @@ -308,6 +377,8 @@ pub trait PersistentLayer: Layer { fn file_size(&self) -> Option; fn info(&self, reset: Option) -> HistoricLayerInfo; + + fn access_stats(&self) -> &LayerAccessStats; } pub fn downcast_remote_layer( diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 67a4064cf5..e28bad3683 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -447,6 +447,10 @@ impl PersistentLayer for DeltaLayer { ret } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl DeltaLayer { @@ -578,6 +582,7 @@ impl DeltaLayer { tenant_id: TenantId, filename: &DeltaFileName, file_size: u64, + existing_access_stats: Option, ) -> DeltaLayer { DeltaLayer { path_or_conf: PathOrConf::Conf(conf), @@ -586,7 +591,7 @@ impl DeltaLayer { key_range: filename.key_range.clone(), lsn_range: filename.lsn_range.clone(), file_size, - access_stats: LayerAccessStats::default(), + access_stats: existing_access_stats.unwrap_or_default(), inner: RwLock::new(DeltaLayerInner { loaded: false, file: None, diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 81bf79c0ce..2ce3a4bf63 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -258,6 +258,10 @@ impl PersistentLayer for ImageLayer { ret } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl ImageLayer { @@ -381,6 +385,7 @@ impl ImageLayer { tenant_id: TenantId, filename: &ImageFileName, file_size: u64, + existing_access_stats: Option, ) -> ImageLayer { ImageLayer { path_or_conf: PathOrConf::Conf(conf), @@ -389,7 +394,7 @@ impl ImageLayer { key_range: filename.key_range.clone(), lsn: filename.lsn, file_size, - access_stats: LayerAccessStats::default(), + access_stats: existing_access_stats.unwrap_or_default(), inner: RwLock::new(ImageLayerInner { loaded: false, file: None, diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index dc1f61cc20..3b55ce6c70 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -18,7 +18,10 @@ use utils::{ use super::filename::{DeltaFileName, ImageFileName, LayerFileName}; use super::image_layer::ImageLayer; -use super::{DeltaLayer, LayerAccessStatsReset, LayerIter, LayerKeyIter, PersistentLayer}; +use super::{ + DeltaLayer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter, + LayerResidenceStatus, PersistentLayer, +}; #[derive(Debug)] pub struct RemoteLayer { @@ -35,6 +38,8 @@ pub struct RemoteLayer { is_incremental: bool, + access_stats: LayerAccessStats, + pub(crate) ongoing_download: Arc, } @@ -150,7 +155,7 @@ impl PersistentLayer for RemoteLayer { lsn_start: lsn_range.start, lsn_end: lsn_range.end, remote: true, - access_stats: None, // remote layer doesn't get accessed + access_stats: Some(self.access_stats.to_api_model()), } } else { HistoricLayerInfo::Image { @@ -159,10 +164,14 @@ impl PersistentLayer for RemoteLayer { key_end: key_range.end, lsn_start: lsn_range.start, remote: true, - access_stats: None, // remote layer doesn't get accessed + access_stats: Some(self.access_stats.to_api_model()), } } } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl RemoteLayer { @@ -171,6 +180,7 @@ impl RemoteLayer { timelineid: TimelineId, fname: &ImageFileName, layer_metadata: &LayerFileMetadata, + existing_access_stats: Option, ) -> RemoteLayer { RemoteLayer { tenantid, @@ -182,6 +192,7 @@ impl RemoteLayer { file_name: fname.to_owned().into(), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), + access_stats: existing_access_stats.unwrap_or_default(), } } @@ -190,6 +201,8 @@ impl RemoteLayer { timelineid: TimelineId, fname: &DeltaFileName, layer_metadata: &LayerFileMetadata, + + existing_access_stats: Option, ) -> RemoteLayer { RemoteLayer { tenantid, @@ -201,6 +214,7 @@ impl RemoteLayer { file_name: fname.to_owned().into(), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), + access_stats: existing_access_stats.unwrap_or_default(), } } @@ -221,6 +235,10 @@ impl RemoteLayer { self.tenantid, &fname, file_size, + Some( + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::resident()), + ), )) } else { let fname = ImageFileName { @@ -233,6 +251,10 @@ impl RemoteLayer { self.tenantid, &fname, file_size, + Some( + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::resident()), + ), )) } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index ffca871769..4a8927fca5 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -71,7 +71,9 @@ use walreceiver::spawn_connection_manager_task; use super::layer_map::BatchedUpdates; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; -use super::storage_layer::{DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset}; +use super::storage_layer::{ + DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset, LayerResidenceStatus, +}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum FlushLoopState { @@ -877,12 +879,22 @@ impl Timeline { self.timeline_id, &image_name, &layer_metadata, + Some( + local_layer + .access_stats() + .clone_for_residence_change(LayerResidenceStatus::evicted()), + ), ), LayerFileName::Delta(delta_name) => RemoteLayer::new_delta( self.tenant_id, self.timeline_id, &delta_name, &layer_metadata, + Some( + local_layer + .access_stats() + .clone_for_residence_change(LayerResidenceStatus::evicted()), + ), ), #[cfg(test)] LayerFileName::Test(_) => unreachable!(), @@ -1159,6 +1171,7 @@ impl Timeline { self.tenant_id, &imgfilename, file_size, + None, ); trace!("found layer {}", layer.path().display()); @@ -1190,6 +1203,7 @@ impl Timeline { self.tenant_id, &deltafilename, file_size, + None, ); trace!("found layer {}", layer.path().display()); @@ -1327,6 +1341,7 @@ impl Timeline { self.timeline_id, imgfilename, &remote_layer_metadata, + None, ); let remote_layer = Arc::new(remote_layer); @@ -1351,6 +1366,7 @@ impl Timeline { self.timeline_id, deltafilename, &remote_layer_metadata, + None, ); let remote_layer = Arc::new(remote_layer); updates.insert_historic(remote_layer);