maintain access stats for historic layers

This patch adds basic access statistics for historic layers
and exposes them in the management API's `LayerMapInfo`.

We record the accesses in the `{Delta,Image}Layer::load()` function
because it's the common path of
* page_service (`Timline::get_reconstruct_data()`)
* Compaction (`PersistentLayer::iter()` and `PersistentLayer::key_iter()`)

There is a killswitch in pageserver.toml to disable statistics gathering
to support before/after benchmarks. With the killswitch enabled, the
overhead of this feature is the function call and the atomic load.

Preceding PRs:
- #3504 add RequestContext plumbing for layer access stats

This is PR https://github.com/neondatabase/neon/pull/3496

Cargo.lock

@@ -1223,6 +1223,47 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "enum-map"
+version = "2.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50c25992259941eb7e57b936157961b217a4fc8597829ddef0596d6c3cd86e1a"
+dependencies = [
+ "enum-map-derive",
+]
+
+[[package]]
+name = "enum-map-derive"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "enumset"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19be8061a06ab6f3a6cf21106c873578bf01bd42ad15e0311a9c76161cb1c753"
+dependencies = [
+ "enumset_derive",
+]
+
+[[package]]
+name = "enumset_derive"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.10.0"
@@ -2320,6 +2361,8 @@ dependencies = [
  "crc32c",
  "criterion",
  "crossbeam-utils",
+ "enum-map",
+ "enumset",
  "fail",
  "futures",
  "git-version",
@@ -2352,6 +2395,8 @@ dependencies = [
  "serde_with",
  "signal-hook",
  "storage_broker",
+ "strum",
+ "strum_macros",
  "svg_fmt",
  "tempfile",
  "tenant_size_model",
@@ -2376,6 +2421,7 @@ dependencies = [
  "byteorder",
  "bytes",
  "const_format",
+ "enum-map",
  "postgres_ffi",
  "serde",
  "serde_with",

libs/pageserver_api/Cargo.toml

@@ -15,3 +15,4 @@ utils.workspace = true
 postgres_ffi.workspace = true
 
 workspace_hack.workspace = true
+enum-map = "2.4.2"

libs/pageserver_api/src/models.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::HashMap,
     fmt,
     num::{NonZeroU64, NonZeroUsize},
     ops::Range,
@@ -237,7 +238,30 @@ pub struct LayerMapInfo {
     pub historic_layers: Vec<HistoricLayerInfo>,
 }
 
+#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, enum_map::Enum)]
+#[repr(usize)]
+pub enum LayerAccessKind {
+    GetValueReconstructData,
+    Iter,
+    KeyIter,
+    Dump,
+}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LayerAccessStatFullDetails {
+    pub when_millis_since_epoch: u128,
+    pub task_kind: &'static str,
+    pub access_kind: LayerAccessKind,
+}
+
 #[derive(Debug, Clone, Serialize)]
+pub struct LayerAccessStats {
+    pub access_count_by_access_kind: HashMap<LayerAccessKind, u64>,
+    pub task_kind_access_flag: Vec<&'static str>,
+    pub first: Option<LayerAccessStatFullDetails>,
+    pub most_recent: Vec<LayerAccessStatFullDetails>,
+}
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
 #[serde_as]
 #[serde(tag = "kind")]
 pub enum InMemoryLayerInfo {
@@ -268,6 +292,7 @@ pub enum HistoricLayerInfo {
         #[serde_as(as = "DisplayFromStr")]
         lsn_end: Lsn,
         remote: bool,
+        access_stats: Option<LayerAccessStats>,
     },
     Image {
         layer_file_name: String,
@@ -278,6 +303,7 @@ pub enum HistoricLayerInfo {
         #[serde_as(as = "DisplayFromStr")]
         lsn_start: Lsn,
         remote: bool,
+        access_stats: Option<LayerAccessStats>,
     },
 }
 

pageserver/Cargo.toml

@@ -67,6 +67,10 @@ utils.workspace = true
 workspace_hack.workspace = true
 reqwest.workspace = true
 rpds.workspace = true
+enumset = "1.0.12"
+enum-map = "2.4.2"
+strum.workspace = true
+strum_macros.workspace = true
 
 [dev-dependencies]
 criterion.workspace = true

pageserver/src/bin/pageserver.rs

@@ -1,12 +1,13 @@
 //! Main entry point for the Page Server executable.
 
 use std::env::{var, VarError};
-use std::sync::Arc;
+use std::sync::{atomic, Arc};
 use std::{env, ops::ControlFlow, path::Path, str::FromStr};
 
 use anyhow::{anyhow, Context};
 use clap::{Arg, ArgAction, Command};
 use fail::FailScenario;
+use pageserver::tenant::storage_layer;
 use remote_storage::GenericRemoteStorage;
 use tracing::*;
 
@@ -85,6 +86,9 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
+    storage_layer::LAYER_ACCESS_STATS_KILLSWITCH
+        .store(conf.layer_access_stats_disable, atomic::Ordering::SeqCst);
+
     // initialize sentry if SENTRY_DSN is provided
     let _sentry_guard = init_sentry(
         Some(GIT_VERSION.into()),

pageserver/src/config.rs

@@ -160,6 +160,7 @@ pub struct PageServerConf {
     pub test_remote_failures: u64,
 
     pub ondemand_download_behavior_treat_error_as_warn: bool,
+    pub layer_access_stats_disable: bool,
 }
 
 /// We do not want to store this in a PageServerConf because the latter may be logged
@@ -226,6 +227,7 @@ struct PageServerConfigBuilder {
     test_remote_failures: BuilderValue<u64>,
 
     ondemand_download_behavior_treat_error_as_warn: BuilderValue<bool>,
+    layer_access_stats_disable: BuilderValue<bool>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -273,6 +275,7 @@ impl Default for PageServerConfigBuilder {
             test_remote_failures: Set(0),
 
             ondemand_download_behavior_treat_error_as_warn: Set(false),
+            layer_access_stats_disable: Set(false),
         }
     }
 }
@@ -377,6 +380,10 @@ impl PageServerConfigBuilder {
             BuilderValue::Set(ondemand_download_behavior_treat_error_as_warn);
     }
 
+    pub fn layer_access_stats_disable(&mut self, val: bool) {
+        self.layer_access_stats_disable = BuilderValue::Set(val);
+    }
+
     pub fn build(self) -> anyhow::Result<PageServerConf> {
         Ok(PageServerConf {
             listen_pg_addr: self
@@ -441,6 +448,9 @@ impl PageServerConfigBuilder {
                 .ok_or(anyhow!(
                     "missing ondemand_download_behavior_treat_error_as_warn"
                 ))?,
+            layer_access_stats_disable: self
+                .layer_access_stats_disable
+                .ok_or(anyhow!("missing layer_acccess_stats_disable"))?,
         })
     }
 }
@@ -620,6 +630,7 @@ impl PageServerConf {
                     builder.synthetic_size_calculation_interval(parse_toml_duration(key, item)?),
                 "test_remote_failures" => builder.test_remote_failures(parse_toml_u64(key, item)?),
                 "ondemand_download_behavior_treat_error_as_warn" => builder.ondemand_download_behavior_treat_error_as_warn(parse_toml_bool(key, item)?),
+                "layer_access_stats_disable" => builder.layer_access_stats_disable(parse_toml_bool(key, item)?),
                 _ => bail!("unrecognized pageserver option '{key}'"),
             }
         }
@@ -745,6 +756,7 @@ impl PageServerConf {
             synthetic_size_calculation_interval: Duration::from_secs(60),
             test_remote_failures: 0,
             ondemand_download_behavior_treat_error_as_warn: false,
+            layer_access_stats_disable: false,
         }
     }
 }
@@ -934,6 +946,7 @@ log_format = 'json'
                 )?,
                 test_remote_failures: 0,
                 ondemand_download_behavior_treat_error_as_warn: false,
+                layer_access_stats_disable: false,
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -982,6 +995,7 @@ log_format = 'json'
                 synthetic_size_calculation_interval: Duration::from_secs(333),
                 test_remote_failures: 0,
                 ondemand_download_behavior_treat_error_as_warn: false,
+                layer_access_stats_disable: false,
             },
             "Should be able to parse all basic config values correctly"
         );

pageserver/src/http/routes.rs

@@ -18,6 +18,7 @@ use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::task_mgr::TaskKind;
 use crate::tenant::config::TenantConfOpt;
 use crate::tenant::mgr::TenantMapInsertError;
+use crate::tenant::storage_layer::LayerAccessStatsReset;
 use crate::tenant::{PageReconstructError, Timeline};
 use crate::{config::PageServerConf, tenant::mgr};
 use utils::{
@@ -535,6 +536,8 @@ async fn layer_map_info_handler(request: Request<Body>) -> Result<Response<Body>
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
+    let reset: Option<LayerAccessStatsReset> = parse_query_param(&request, "reset")?;
+
     let tenant = mgr::get_tenant(tenant_id, true)
         .await
         .map_err(ApiError::NotFound)?;
@@ -542,7 +545,7 @@ async fn layer_map_info_handler(request: Request<Body>) -> Result<Response<Body>
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
 
-    let layer_map_info = timeline.layer_map_info();
+    let layer_map_info = timeline.layer_map_info(reset);
 
     json_response(StatusCode::OK, layer_map_info)
 }

pageserver/src/task_mgr.rs

@@ -169,7 +169,14 @@ task_local! {
 /// Note that we don't try to limit how many task of a certain kind can be running
 /// at the same time.
 ///
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+#[derive(
+    Debug,
+    // NB: enumset::EnumSetType derives PartialEq, Eq, Clone, Copy
+    enumset::EnumSetType,
+    serde::Serialize,
+    serde::Deserialize,
+    strum_macros::IntoStaticStr,
+)]
 pub enum TaskKind {
     // Pageserver startup, i.e., `main`
     Startup,

pageserver/src/tenant/storage_layer.rs

@@ -6,14 +6,21 @@ mod image_layer;
 mod inmemory_layer;
 mod remote_layer;
 
+use crate::task_mgr::TaskKind;
 use crate::walrecord::NeonWalRecord;
 use crate::{context::RequestContext, repository::Value};
 use anyhow::Result;
 use bytes::Bytes;
+use enum_map::EnumMap;
+use enumset::EnumSet;
+use pageserver_api::models::LayerAccessKind;
 use pageserver_api::models::{HistoricLayerInfo, Key};
+use std::collections::VecDeque;
 use std::ops::Range;
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::atomic::{self, AtomicBool};
+use std::sync::{Arc, RwLock};
+use std::time::{SystemTime, UNIX_EPOCH};
 
 use utils::{
     id::{TenantId, TimelineId},
@@ -81,6 +88,103 @@ pub enum ValueReconstructResult {
     Missing,
 }
 
+#[derive(Default)]
+pub struct LayerAccessStats(RwLock<LayerAccessStatsInner>);
+
+#[derive(Clone)]
+struct LayerAccessStatFullDetails {
+    when: SystemTime,
+    task_kind: TaskKind,
+    access_kind: LayerAccessKind,
+}
+
+#[derive(Default)]
+struct LayerAccessStatsInner {
+    first_access: Option<LayerAccessStatFullDetails>,
+    count_by_access_kind: EnumMap<LayerAccessKind, u64>,
+    task_kind_flag: EnumSet<TaskKind>,
+    last_accesses: VecDeque<LayerAccessStatFullDetails>,
+}
+
+#[derive(Clone, Copy, strum_macros::EnumString)]
+pub enum LayerAccessStatsReset {
+    JustTaskKindFlags,
+    AllStats,
+}
+
+impl LayerAccessStatFullDetails {
+    fn to_api_model(&self) -> pageserver_api::models::LayerAccessStatFullDetails {
+        let Self {
+            when,
+            task_kind,
+            access_kind,
+        } = self;
+        pageserver_api::models::LayerAccessStatFullDetails {
+            when_millis_since_epoch: when
+                .duration_since(UNIX_EPOCH)
+                .expect("better to die in this unlikely case than report false stats")
+                .as_millis(),
+            task_kind: task_kind.into(), // into static str, powered by strum_macros
+            access_kind: *access_kind,
+        }
+    }
+}
+
+pub static LAYER_ACCESS_STATS_KILLSWITCH: AtomicBool = AtomicBool::new(false);
+
+impl LayerAccessStats {
+    fn record_access(&self, access_kind: LayerAccessKind, task_kind: TaskKind) {
+        if LAYER_ACCESS_STATS_KILLSWITCH.load(atomic::Ordering::SeqCst) {
+            return;
+        }
+        let mut inner = self.0.write().unwrap();
+        let this_access = LayerAccessStatFullDetails {
+            when: SystemTime::now(),
+            task_kind,
+            access_kind,
+        };
+        inner
+            .first_access
+            .get_or_insert_with(|| this_access.clone());
+        inner.count_by_access_kind[access_kind] += 1;
+        inner.task_kind_flag |= task_kind;
+        inner.last_accesses.push_front(this_access);
+        inner.last_accesses.truncate(10); // TODO make runtime-configurable
+    }
+    fn reset(&self, what: LayerAccessStatsReset) {
+        let mut inner = self.0.write().unwrap();
+        match what {
+            LayerAccessStatsReset::JustTaskKindFlags => {
+                inner.task_kind_flag.clear();
+            }
+            LayerAccessStatsReset::AllStats => {
+                *inner = LayerAccessStatsInner::default();
+            }
+        }
+    }
+    fn to_api_model(&self) -> pageserver_api::models::LayerAccessStats {
+        let inner = self.0.read().unwrap();
+        let LayerAccessStatsInner {
+            first_access,
+            count_by_access_kind,
+            task_kind_flag,
+            last_accesses,
+        } = &*inner;
+        pageserver_api::models::LayerAccessStats {
+            access_count_by_access_kind: count_by_access_kind
+                .iter()
+                .map(|(kind, count)| (kind, *count))
+                .collect(),
+            task_kind_access_flag: task_kind_flag
+                .iter()
+                .map(|task_kind| task_kind.into()) // into static str, powered by strum_macros
+                .collect(),
+            first: first_access.as_ref().map(|a| a.to_api_model()),
+            most_recent: last_accesses.iter().map(|a| a.to_api_model()).collect(),
+        }
+    }
+}
+
 /// Supertrait of the [`Layer`] trait that captures the bare minimum interface
 /// required by [`LayerMap`].
 pub trait Layer: Send + Sync {
@@ -188,7 +292,7 @@ pub trait PersistentLayer: Layer {
     /// current_physical_size is computed as the som of this value.
     fn file_size(&self) -> Option<u64>;
 
-    fn info(&self) -> HistoricLayerInfo;
+    fn info(&self, reset: Option<LayerAccessStatsReset>) -> HistoricLayerInfo;
 }
 
 pub fn downcast_remote_layer(

pageserver/src/tenant/storage_layer/delta_layer.rs

@@ -37,7 +37,7 @@ use crate::virtual_file::VirtualFile;
 use crate::{walrecord, TEMP_FILE_SUFFIX};
 use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION};
 use anyhow::{bail, ensure, Context, Result};
-use pageserver_api::models::{HistoricLayerInfo, Key, KEY_SIZE};
+use pageserver_api::models::{HistoricLayerInfo, Key, LayerAccessKind, KEY_SIZE};
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
@@ -55,7 +55,10 @@ use utils::{
     lsn::Lsn,
 };
 
-use super::{DeltaFileName, Layer, LayerFileName, LayerIter, LayerKeyIter, PathOrConf};
+use super::{
+    DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, LayerFileName, LayerIter,
+    LayerKeyIter, PathOrConf,
+};
 
 ///
 /// Header stored in the beginning of the file
@@ -185,6 +188,8 @@ pub struct DeltaLayer {
 
     pub file_size: u64,
 
+    access_stats: LayerAccessStats,
+
     inner: RwLock<DeltaLayerInner>,
 }
 
@@ -231,7 +236,7 @@ impl Layer for DeltaLayer {
             return Ok(());
         }
 
-        let inner = self.load(ctx)?;
+        let inner = self.load(LayerAccessKind::Dump, ctx)?;
 
         println!(
             "index_start_blk: {}, root {}",
@@ -304,7 +309,7 @@ impl Layer for DeltaLayer {
 
         {
             // Open the file and lock the metadata in memory
-            let inner = self.load(ctx)?;
+            let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?;
 
             // Scan the page versions backwards, starting from `lsn`.
             let file = inner.file.as_ref().unwrap();
@@ -395,7 +400,9 @@ impl PersistentLayer for DeltaLayer {
     }
 
     fn iter(&self, ctx: &RequestContext) -> Result<LayerIter<'_>> {
-        let inner = self.load(ctx).context("load delta layer")?;
+        let inner = self
+            .load(LayerAccessKind::KeyIter, ctx)
+            .context("load delta layer")?;
         Ok(match DeltaValueIter::new(inner) {
             Ok(iter) => Box::new(iter),
             Err(err) => Box::new(std::iter::once(Err(err))),
@@ -403,7 +410,7 @@ impl PersistentLayer for DeltaLayer {
     }
 
     fn key_iter(&self, ctx: &RequestContext) -> Result<LayerKeyIter<'_>> {
-        let inner = self.load(ctx)?;
+        let inner = self.load(LayerAccessKind::KeyIter, ctx)?;
         Ok(Box::new(
             DeltaKeyIter::new(inner).context("Layer index is corrupted")?,
         ))
@@ -419,19 +426,26 @@ impl PersistentLayer for DeltaLayer {
         Some(self.file_size)
     }
 
-    fn info(&self) -> HistoricLayerInfo {
+    fn info(&self, reset: Option<LayerAccessStatsReset>) -> HistoricLayerInfo {
         let layer_file_name = self.filename().file_name();
         let key_range = self.get_key_range();
         let lsn_range = self.get_lsn_range();
 
-        HistoricLayerInfo::Delta {
+        let ret = HistoricLayerInfo::Delta {
             layer_file_name,
             key_start: key_range.start,
             key_end: key_range.end,
             lsn_start: lsn_range.start,
             lsn_end: lsn_range.end,
             remote: false,
+            access_stats: Some(self.access_stats.to_api_model()),
+        };
+
+        if let Some(reset) = reset {
+            self.access_stats.reset(reset);
         }
+
+        ret
     }
 }
 
@@ -477,7 +491,13 @@ impl DeltaLayer {
     /// Open the underlying file and read the metadata into memory, if it's
     /// not loaded already.
     ///
-    fn load(&self, _ctx: &RequestContext) -> Result<RwLockReadGuard<DeltaLayerInner>> {
+    fn load(
+        &self,
+        access_kind: LayerAccessKind,
+        ctx: &RequestContext,
+    ) -> Result<RwLockReadGuard<DeltaLayerInner>> {
+        self.access_stats
+            .record_access(access_kind, ctx.task_kind());
         loop {
             // Quick exit if already loaded
             let inner = self.inner.read().unwrap();
@@ -566,6 +586,7 @@ impl DeltaLayer {
             key_range: filename.key_range.clone(),
             lsn_range: filename.lsn_range.clone(),
             file_size,
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,
@@ -595,6 +616,7 @@ impl DeltaLayer {
             key_range: summary.key_range,
             lsn_range: summary.lsn_range,
             file_size: metadata.len(),
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,
@@ -765,6 +787,7 @@ impl DeltaLayerWriterInner {
             key_range: self.key_start..key_end,
             lsn_range: self.lsn_range.clone(),
             file_size: metadata.len(),
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(DeltaLayerInner {
                 loaded: false,
                 file: None,

pageserver/src/tenant/storage_layer/image_layer.rs

@@ -33,7 +33,7 @@ use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION, TEMP_FILE_SUFFIX};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
 use hex;
-use pageserver_api::models::{HistoricLayerInfo, Key, KEY_SIZE};
+use pageserver_api::models::{HistoricLayerInfo, Key, LayerAccessKind, KEY_SIZE};
 use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
@@ -52,7 +52,7 @@ use utils::{
 };
 
 use super::filename::{ImageFileName, LayerFileName, PathOrConf};
-use super::{Layer, LayerIter};
+use super::{Layer, LayerAccessStats, LayerAccessStatsReset, LayerIter};
 
 ///
 /// Header stored in the beginning of the file
@@ -111,6 +111,8 @@ pub struct ImageLayer {
     // This entry contains an image of all pages as of this LSN
     pub lsn: Lsn,
 
+    access_stats: LayerAccessStats,
+
     inner: RwLock<ImageLayerInner>,
 }
 
@@ -154,7 +156,7 @@ impl Layer for ImageLayer {
             return Ok(());
         }
 
-        let inner = self.load(ctx)?;
+        let inner = self.load(LayerAccessKind::Dump, ctx)?;
         let file = inner.file.as_ref().unwrap();
         let tree_reader =
             DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file);
@@ -181,7 +183,7 @@ impl Layer for ImageLayer {
         assert!(lsn_range.start >= self.lsn);
         assert!(lsn_range.end >= self.lsn);
 
-        let inner = self.load(ctx)?;
+        let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?;
 
         let file = inner.file.as_ref().unwrap();
         let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file);
@@ -236,18 +238,25 @@ impl PersistentLayer for ImageLayer {
         Some(self.file_size)
     }
 
-    fn info(&self) -> HistoricLayerInfo {
+    fn info(&self, reset: Option<LayerAccessStatsReset>) -> HistoricLayerInfo {
         let layer_file_name = self.filename().file_name();
         let key_range = self.get_key_range();
         let lsn_range = self.get_lsn_range();
 
-        HistoricLayerInfo::Image {
+        let ret = HistoricLayerInfo::Image {
             layer_file_name,
             key_start: key_range.start,
             key_end: key_range.end,
             lsn_start: lsn_range.start,
             remote: false,
+            access_stats: Some(self.access_stats.to_api_model()),
+        };
+
+        if let Some(reset) = reset {
+            self.access_stats.reset(reset);
         }
+
+        ret
     }
 }
 
@@ -286,7 +295,13 @@ impl ImageLayer {
     /// Open the underlying file and read the metadata into memory, if it's
     /// not loaded already.
     ///
-    fn load(&self, _ctx: &RequestContext) -> Result<RwLockReadGuard<ImageLayerInner>> {
+    fn load(
+        &self,
+        access_kind: LayerAccessKind,
+        ctx: &RequestContext,
+    ) -> Result<RwLockReadGuard<ImageLayerInner>> {
+        self.access_stats
+            .record_access(access_kind, ctx.task_kind());
         loop {
             // Quick exit if already loaded
             let inner = self.inner.read().unwrap();
@@ -374,6 +389,7 @@ impl ImageLayer {
             key_range: filename.key_range.clone(),
             lsn: filename.lsn,
             file_size,
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(ImageLayerInner {
                 loaded: false,
                 file: None,
@@ -401,6 +417,7 @@ impl ImageLayer {
             key_range: summary.key_range,
             lsn: summary.lsn,
             file_size: metadata.len(),
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(ImageLayerInner {
                 file: None,
                 loaded: false,
@@ -560,6 +577,7 @@ impl ImageLayerWriterInner {
             key_range: self.key_range.clone(),
             lsn: self.lsn,
             file_size: metadata.len(),
+            access_stats: LayerAccessStats::default(),
             inner: RwLock::new(ImageLayerInner {
                 loaded: false,
                 file: None,

pageserver/src/tenant/storage_layer/remote_layer.rs

@@ -18,7 +18,7 @@ use utils::{
 
 use super::filename::{DeltaFileName, ImageFileName, LayerFileName};
 use super::image_layer::ImageLayer;
-use super::{DeltaLayer, LayerIter, LayerKeyIter, PersistentLayer};
+use super::{DeltaLayer, LayerAccessStatsReset, LayerIter, LayerKeyIter, PersistentLayer};
 
 #[derive(Debug)]
 pub struct RemoteLayer {
@@ -137,7 +137,7 @@ impl PersistentLayer for RemoteLayer {
         self.layer_metadata.file_size()
     }
 
-    fn info(&self) -> HistoricLayerInfo {
+    fn info(&self, _reset: Option<LayerAccessStatsReset>) -> HistoricLayerInfo {
         let layer_file_name = self.filename().file_name();
         let key_range = self.get_key_range();
         let lsn_range = self.get_lsn_range();
@@ -150,6 +150,7 @@ impl PersistentLayer for RemoteLayer {
                 lsn_start: lsn_range.start,
                 lsn_end: lsn_range.end,
                 remote: true,
+                access_stats: None, // remote layer doesn't get accessed
             }
         } else {
             HistoricLayerInfo::Image {
@@ -158,6 +159,7 @@ impl PersistentLayer for RemoteLayer {
                 key_end: key_range.end,
                 lsn_start: lsn_range.start,
                 remote: true,
+                access_stats: None, // remote layer doesn't get accessed
             }
         }
     }

pageserver/src/tenant/timeline.rs

@@ -71,7 +71,7 @@ use walreceiver::spawn_connection_manager_task;
 use super::layer_map::BatchedUpdates;
 use super::remote_timeline_client::index::IndexPart;
 use super::remote_timeline_client::RemoteTimelineClient;
-use super::storage_layer::{DeltaLayer, ImageLayer, Layer};
+use super::storage_layer::{DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset};
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 enum FlushLoopState {
@@ -820,7 +820,7 @@ impl Timeline {
         self.state.subscribe()
     }
 
-    pub fn layer_map_info(&self) -> LayerMapInfo {
+    pub fn layer_map_info(&self, reset: Option<LayerAccessStatsReset>) -> LayerMapInfo {
         let mut in_memory_layers = Vec::new();
         let layer_map = self.layers.read().unwrap();
         if let Some(open_layer) = &layer_map.open_layer {
@@ -832,7 +832,7 @@ impl Timeline {
 
         let mut historic_layers = Vec::new();
         for historic_layer in layer_map.iter_historic_layers() {
-            historic_layers.push(historic_layer.info());
+            historic_layers.push(historic_layer.info(reset));
         }
 
         LayerMapInfo {