From 6b76e1c52601c145a6bd45b7a4866507f585940d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 26 Feb 2025 20:13:27 +0200 Subject: [PATCH] Add lazy_slru_download compute feature flag --- compute_tools/src/compute.rs | 29 ++++---------------- libs/compute_api/src/spec.rs | 3 ++ pageserver/src/basebackup.rs | 9 ++++-- pageserver/src/page_service.rs | 50 +++++++++++++++++++++++++++++----- 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index c7b4bdd240..818563c6ee 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -897,30 +897,13 @@ impl ComputeNode { let mut client = config.connect(NoTls)?; let pageserver_connect_micros = start_time.elapsed().as_micros() as u64; + let replica = if spec.spec.mode != ComputeMode::Primary { " --replica" } else { "" }; + let lazy_slru_downloand = if spec.features.contains(&ComputeFeature::LazySlruDownload) { " --lazy-slru-download" } else { "" }; let basebackup_cmd = match lsn { - Lsn(0) => { - if spec.spec.mode != ComputeMode::Primary { - format!( - "basebackup {} {} --gzip --replica", - spec.tenant_id, spec.timeline_id - ) - } else { - format!("basebackup {} {} --gzip", spec.tenant_id, spec.timeline_id) - } - } - _ => { - if spec.spec.mode != ComputeMode::Primary { - format!( - "basebackup {} {} {} --gzip --replica", - spec.tenant_id, spec.timeline_id, lsn - ) - } else { - format!( - "basebackup {} {} {} --gzip", - spec.tenant_id, spec.timeline_id, lsn - ) - } - } + Lsn(0) => format!("basebackup {} {} --gzip{}{}", + spec.tenant_id, spec.timeline_id, replica, lazy_slru_dpownload) + _ => format!("basebackup {} {} {} --gzip{}{}", + spec.tenant_id, spec.timeline_id, lsn, replica, lazy_slru_dpownload) }; let copyreader = client.copy_out(basebackup_cmd.as_str())?; diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index 5e67ccce00..ea85e4cc52 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -183,6 +183,9 @@ pub enum ComputeFeature { /// track short-lived connections as user activity. ActivityMonitorExperimental, + /// Download larger SLU files on demand + LazySlruDownload, + /// This is a special feature flag that is used to represent unknown feature flags. /// Basically all unknown to enum flags are represented as this one. See unit test /// `parse_unknown_features()` for more details. diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index 3510ccb529..26571f111e 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -80,6 +80,7 @@ pub async fn send_basebackup_tarball<'a, W>( prev_lsn: Option, full_backup: bool, replica: bool, + lazy_slru_download: bool, ctx: &'a RequestContext, ) -> Result<(), BasebackupError> where @@ -131,8 +132,8 @@ where }; info!( - "taking basebackup lsn={}, prev_lsn={} (full_backup={}, replica={})", - backup_lsn, prev_lsn, full_backup, replica + "taking basebackup lsn={}, prev_lsn={} (full_backup={}, replica={}, lazy_slru_download={})", + backup_lsn, prev_lsn, full_backup, replica, lazy_slru_download ); let basebackup = Basebackup { @@ -142,6 +143,7 @@ where prev_record_lsn: prev_lsn, full_backup, replica, + lazy_slru_dpownload, ctx, io_concurrency: IoConcurrency::spawn_from_conf( timeline.conf, @@ -170,6 +172,7 @@ where prev_record_lsn: Lsn, full_backup: bool, replica: bool, + lazy_slru_download: bool, ctx: &'a RequestContext, io_concurrency: IoConcurrency, } @@ -308,7 +311,7 @@ where self.timeline.pg_version, )?; - let lazy_slru_download = self.timeline.get_lazy_slru_download() && !self.full_backup; + let lazy_slru_download = (self.self.lazy_slru_download || timeline.get_lazy_slru_download()) && !self.full_backup; let pgversion = self.timeline.pg_version; let subdirs = dispatch_pgversion!(pgversion, &pgv::bindings::PGDATA_SUBDIRS[..]); diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 560ac75f4a..c3d1ba2b31 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -2394,6 +2394,7 @@ impl PageServerHandler { full_backup: bool, gzip: bool, replica: bool, + lazy_slru_download: bool, ctx: &RequestContext, ) -> Result<(), QueryError> where @@ -2461,6 +2462,7 @@ impl PageServerHandler { prev_lsn, full_backup, replica, + lazy_slru_download, &ctx, ) .await @@ -2484,6 +2486,7 @@ impl PageServerHandler { prev_lsn, full_backup, replica, + lazy_slru_download, &ctx, ) .await @@ -2501,6 +2504,7 @@ impl PageServerHandler { prev_lsn, full_backup, replica, + lazy_slru_download, &ctx, ) .await @@ -2550,7 +2554,7 @@ impl PageServerHandler { } } -/// `basebackup tenant timeline [lsn] [--gzip] [--replica]` +/// `basebackup tenant timeline [lsn] [--gzip] [--replica]`[--lazy-sru-download] #[derive(Debug, Clone, Eq, PartialEq)] struct BaseBackupCmd { tenant_id: TenantId, @@ -2558,6 +2562,7 @@ struct BaseBackupCmd { lsn: Option, gzip: bool, replica: bool, + lazy_slru_download: bool, } /// `fullbackup tenant timeline [lsn] [prev_lsn]` @@ -2690,6 +2695,7 @@ impl BaseBackupCmd { let mut gzip = false; let mut replica = false; + let mut lazy_slru_download = false; for ¶m in ¶meters[flags_parse_from..] { match param { @@ -2705,6 +2711,12 @@ impl BaseBackupCmd { } replica = true } + "--lazy-slru-download" => { + if lazy_slru_download { + bail!("duplicate parameter for basebackup command: {param}") + } + lazy_slru_download = true + } _ => bail!("invalid parameter for basebackup command: {param}"), } } @@ -2714,6 +2726,7 @@ impl BaseBackupCmd { lsn, gzip, replica, + lazy_slru_download, }) } } @@ -2928,6 +2941,7 @@ where lsn, gzip, replica, + lazy_slru_download, }) => { tracing::Span::current() .record("tenant_id", field::display(tenant_id)) @@ -2949,6 +2963,7 @@ where false, gzip, replica, + lazy_slru_download, &ctx, ) .await?; @@ -3120,7 +3135,8 @@ mod tests { timeline_id, lsn: None, gzip: false, - replica: false + replica: false, + lazy_slru_download: false }) ); let cmd = @@ -3132,7 +3148,8 @@ mod tests { timeline_id, lsn: None, gzip: true, - replica: false + replica: false, + lazy_slru_download: false }) ); let cmd = @@ -3144,7 +3161,8 @@ mod tests { timeline_id, lsn: None, gzip: false, - replica: false + replica: false, + lazy_slru_download: false }) ); let cmd = PageServiceCmd::parse(&format!("basebackup {tenant_id} {timeline_id} 0/16ABCDE")) @@ -3156,7 +3174,8 @@ mod tests { timeline_id, lsn: Some(Lsn::from_str("0/16ABCDE").unwrap()), gzip: false, - replica: false + replica: false, + lazy_slru_download: false }) ); let cmd = PageServiceCmd::parse(&format!( @@ -3170,7 +3189,23 @@ mod tests { timeline_id, lsn: None, gzip: true, - replica: true + replica: true, + lazy_slru_download: false + }) + ); + let cmd = PageServiceCmd::parse(&format!( + "basebackup {tenant_id} {timeline_id} --replica --gzip --lazy-slru-download" + )) + .unwrap(); + assert_eq!( + cmd, + PageServiceCmd::BaseBackup(BaseBackupCmd { + tenant_id, + timeline_id, + lsn: None, + gzip: true, + replica: true, + lazy_slru_download: true }) ); let cmd = PageServiceCmd::parse(&format!( @@ -3184,7 +3219,8 @@ mod tests { timeline_id, lsn: Some(Lsn::from_str("0/16ABCDE").unwrap()), gzip: true, - replica: true + replica: true, + lazy_slru_download: false }) ); let cmd = PageServiceCmd::parse(&format!("fullbackup {tenant_id} {timeline_id}")).unwrap();