diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index abf815f07a..6f9d53ba33 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -434,6 +434,11 @@ impl PageServerNode { .map(serde_json::from_str) .transpose() .context("parse `timeline_get_throttle` from json")?, + try_enable_aux_file_v2: settings + .remove("try_enable_aux_file_v2") + .map(|x| x.parse::()) + .transpose() + .context("Failed to parse 'try_enable_aux_file_v2' as bool")?, }; if !settings.is_empty() { bail!("Unrecognized tenant settings: {settings:?}") @@ -552,6 +557,11 @@ impl PageServerNode { .map(serde_json::from_str) .transpose() .context("parse `timeline_get_throttle` from json")?, + try_enable_aux_file_v2: settings + .remove("try_enable_aux_file_v2") + .map(|x| x.parse::()) + .transpose() + .context("Failed to parse 'try_enable_aux_file_v2' as bool")?, } }; diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index f441d1ff1a..7919f9721f 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -303,6 +303,7 @@ pub struct TenantConfig { pub lazy_slru_download: Option, pub timeline_get_throttle: Option, pub image_layer_creation_check_threshold: Option, + pub try_enable_aux_file_v2: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -578,6 +579,9 @@ pub struct TimelineInfo { pub state: TimelineState, pub walreceiver_status: String, + + /// Whether aux file v2 is enabled + pub aux_file_v2: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index 1fb75584fc..00c410fba2 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -74,6 +74,8 @@ struct MetadataCmd { prev_record_lsn: Option, /// Replace latest gc cuttoff latest_gc_cuttoff: Option, + /// Enable aux file v2 storage + aux_file_v2: Option, } #[derive(Parser)] @@ -213,12 +215,14 @@ fn handle_metadata( disk_consistent_lsn, prev_record_lsn, latest_gc_cuttoff, + aux_file_v2, }: &MetadataCmd, ) -> Result<(), anyhow::Error> { let metadata_bytes = std::fs::read(path)?; let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?; println!("Current metadata:\n{meta:?}"); let mut update_meta = false; + // TODO: simplify this part if let Some(disk_consistent_lsn) = disk_consistent_lsn { meta = TimelineMetadata::new( *disk_consistent_lsn, @@ -228,6 +232,7 @@ fn handle_metadata( meta.latest_gc_cutoff_lsn(), meta.initdb_lsn(), meta.pg_version(), + meta.aux_file_v2(), ); update_meta = true; } @@ -240,6 +245,7 @@ fn handle_metadata( meta.latest_gc_cutoff_lsn(), meta.initdb_lsn(), meta.pg_version(), + meta.aux_file_v2() ); update_meta = true; } @@ -252,6 +258,20 @@ fn handle_metadata( *latest_gc_cuttoff, meta.initdb_lsn(), meta.pg_version(), + meta.aux_file_v2(), + ); + update_meta = true; + } + if let Some(aux_file_v2) = aux_file_v2 { + meta = TimelineMetadata::new( + meta.disk_consistent_lsn(), + meta.prev_record_lsn(), + meta.ancestor_timeline(), + meta.ancestor_lsn(), + meta.latest_gc_cutoff_lsn(), + meta.initdb_lsn(), + meta.pg_version(), + *aux_file_v2, ); update_meta = true; } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 81508965b4..58c6a1699c 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -426,6 +426,8 @@ async fn build_timeline_info_common( state, walreceiver_status, + + aux_file_v2: timeline.aux_file_v2.load(std::sync::atomic::Ordering::SeqCst) }; Ok(info) } diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 4a9682dcac..39458aaeec 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -32,7 +32,7 @@ use std::ops::ControlFlow; use std::ops::Range; use strum::IntoEnumIterator; use tokio_util::sync::CancellationToken; -use tracing::{debug, trace, warn}; +use tracing::{debug, info, trace, warn}; use utils::bin_ser::DeserializeError; use utils::vec_map::{VecMap, VecMapOrdering}; use utils::{bin_ser::BeSer, lsn::Lsn}; @@ -1399,6 +1399,31 @@ impl<'a> DatadirModification<'a> { Some(Bytes::copy_from_slice(content)) }; + // TODO: either ensure we don't flip the flag for users with existing AUX files, or do a check there. + let aux_file_v2 = { + let tline_aux_file_v2 = self + .tline + .aux_file_v2 + .load(std::sync::atomic::Ordering::SeqCst); + if tline_aux_file_v2 { + true + } else if self.tline.get_try_enable_aux_file_v2() { + info!( + "enabling aux file v2 support for timeline {}", + self.tline.timeline_id + ); + // The next index part upload will have `aux_file_v2` to `true`. + self.tline + .aux_file_v2 + .store(true, std::sync::atomic::Ordering::SeqCst); + true + } else { + false + } + }; + + let _ = aux_file_v2; // keep this unused until the write path is implemented + let n_files; let mut aux_files = self.tline.aux_files.lock().await; if let Some(mut dir) = aux_files.dir.take() { diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 15be6df637..5e258b763d 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1346,6 +1346,7 @@ impl Tenant { initdb_lsn, initdb_lsn, pg_version, + false, ); self.prepare_new_timeline( new_timeline_id, @@ -3007,6 +3008,7 @@ impl Tenant { *src_timeline.latest_gc_cutoff_lsn.read(), // FIXME: should we hold onto this guard longer? src_timeline.initdb_lsn, src_timeline.pg_version, + src_timeline.aux_file_v2.load(Ordering::SeqCst), ); let uninitialized_timeline = self @@ -3210,6 +3212,7 @@ impl Tenant { pgdata_lsn, pgdata_lsn, pg_version, + false, ); let raw_timeline = self .prepare_new_timeline( @@ -3661,6 +3664,7 @@ pub(crate) mod harness { image_layer_creation_check_threshold: Some( tenant_conf.image_layer_creation_check_threshold, ), + try_enable_aux_file_v2: Some(tenant_conf.try_enable_aux_file_v2), } } } diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index a2bb479f63..42c06d4522 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -369,6 +369,10 @@ pub struct TenantConf { // How much WAL must be ingested before checking again whether a new image layer is required. // Expresed in multiples of checkpoint distance. pub image_layer_creation_check_threshold: u8, + + /// Try enable the aux file v2 storage. Once this is set to true and the tenant writes an AUX file, the + /// pageserver will always use v2 for AUX files and setting this flag to false will be a no-op. + pub try_enable_aux_file_v2: bool, } /// Same as TenantConf, but this struct preserves the information about @@ -464,6 +468,10 @@ pub struct TenantConfOpt { #[serde(skip_serializing_if = "Option::is_none")] pub image_layer_creation_check_threshold: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub try_enable_aux_file_v2: Option, } impl TenantConfOpt { @@ -521,6 +529,9 @@ impl TenantConfOpt { image_layer_creation_check_threshold: self .image_layer_creation_check_threshold .unwrap_or(global_conf.image_layer_creation_check_threshold), + try_enable_aux_file_v2: self + .try_enable_aux_file_v2 + .unwrap_or(global_conf.try_enable_aux_file_v2), } } } @@ -562,6 +573,7 @@ impl Default for TenantConf { lazy_slru_download: false, timeline_get_throttle: crate::tenant::throttle::Config::disabled(), image_layer_creation_check_threshold: DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD, + try_enable_aux_file_v2: false, } } } @@ -636,6 +648,7 @@ impl From for models::TenantConfig { lazy_slru_download: value.lazy_slru_download, timeline_get_throttle: value.timeline_get_throttle.map(ThrottleConfig::from), image_layer_creation_check_threshold: value.image_layer_creation_check_threshold, + try_enable_aux_file_v2: value.try_enable_aux_file_v2, } } } diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 39da713479..0fda370699 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -61,6 +61,7 @@ struct TimelineMetadataBodyV2 { latest_gc_cutoff_lsn: Lsn, initdb_lsn: Lsn, pg_version: u32, + aux_file_v2: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -92,6 +93,7 @@ impl TimelineMetadata { latest_gc_cutoff_lsn: Lsn, initdb_lsn: Lsn, pg_version: u32, + aux_file_v2: bool, ) -> Self { Self { hdr: TimelineMetadataHeader { @@ -107,6 +109,7 @@ impl TimelineMetadata { latest_gc_cutoff_lsn, initdb_lsn, pg_version, + aux_file_v2: if aux_file_v2 { Some(true) } else { None }, }, } } @@ -134,6 +137,7 @@ impl TimelineMetadata { latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn, initdb_lsn: body.initdb_lsn, pg_version: 14, // All timelines created before this version had pg_version 14 + aux_file_v2: None, }; hdr.format_version = METADATA_FORMAT_VERSION; @@ -219,6 +223,10 @@ impl TimelineMetadata { self.body.pg_version } + pub fn aux_file_v2(&self) -> bool { + self.body.aux_file_v2.unwrap_or(false) + } + // Checksums make it awkward to build a valid instance by hand. This helper // provides a TimelineMetadata with a valid checksum in its header. #[cfg(test)] diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index fa7d219fb0..3e4fddd01b 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -40,7 +40,6 @@ use utils::{ vec_map::VecMap, }; -use std::ops::{Deref, Range}; use std::pin::pin; use std::sync::atomic::Ordering as AtomicOrdering; use std::sync::{Arc, Mutex, RwLock, Weak}; @@ -54,6 +53,10 @@ use std::{ cmp::{max, min, Ordering}, ops::ControlFlow, }; +use std::{ + ops::{Deref, Range}, + sync::atomic::AtomicBool, +}; use crate::deletion_queue::DeletionQueueClient; use crate::tenant::timeline::logical_size::CurrentLogicalSize; @@ -382,6 +385,9 @@ pub struct Timeline { /// Keep aux directory cache to avoid it's reconstruction on each update pub(crate) aux_files: tokio::sync::Mutex, + + /// Indicate whether aux file v2 storage is enabled. + pub(crate) aux_file_v2: AtomicBool, } pub struct WalReceiverInfo { @@ -1737,6 +1743,14 @@ const REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE: u64 = 10; // Private functions impl Timeline { + pub(crate) fn get_try_enable_aux_file_v2(&self) -> bool { + let tenant_conf = self.tenant_conf.load(); + tenant_conf + .tenant_conf + .try_enable_aux_file_v2 + .unwrap_or(self.conf.default_tenant_conf.try_enable_aux_file_v2) + } + pub(crate) fn get_lazy_slru_download(&self) -> bool { let tenant_conf = self.tenant_conf.load(); tenant_conf @@ -1987,6 +2001,8 @@ impl Timeline { dir: None, n_deltas: 0, }), + + aux_file_v2: AtomicBool::new(false), }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -2136,6 +2152,11 @@ impl Timeline { let shard = self.get_shard_index(); let this = self.myself.upgrade().expect("&self method holds the arc"); + if let Some(ref index_part) = index_part { + self.aux_file_v2 + .store(index_part.metadata.aux_file_v2(), AtomicOrdering::SeqCst); + } + let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({ move || { let _g = span.entered(); @@ -3606,6 +3627,9 @@ impl Timeline { disk_consistent_lsn, ondisk_prev_record_lsn, *self.latest_gc_cutoff_lsn.read(), + self.initdb_lsn, + self.pg_version, + self.aux_file_v2.load(AtomicOrdering::SeqCst), ); fail_point!("checkpoint-before-saving-metadata", |x| bail!( diff --git a/test_runner/regress/test_aux_files.py b/test_runner/regress/test_aux_files.py new file mode 100644 index 0000000000..c30b02787b --- /dev/null +++ b/test_runner/regress/test_aux_files.py @@ -0,0 +1,63 @@ +from fixtures.log_helper import log +from fixtures.neon_fixtures import ( + NeonEnv, + logical_replication_sync, +) + + +def test_aux_v2_config_switch(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_aux_v2_config_switch", "empty") + endpoint = env.endpoints.create_start( + "test_aux_v2_config_switch", config_lines=["log_statement=all"] + ) + + with env.pageserver.http_client() as client: + tenant_config = client.tenant_config(tenant_id).effective_config + tenant_config["try_enable_aux_file_v2"] = True + client.set_tenant_config(tenant_id, tenant_config) + # aux file v2 is enabled on the write path + assert not client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)[ + "aux_file_v2" + ] + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + cur.execute("create table t(pk integer primary key, payload integer)") + cur.execute( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));" + ) + cur.execute("create publication pub1 for table t, replication_example") + + # now start subscriber, aux files will be created at this point. TODO: find better ways of testing aux files (i.e., neon_test_utils) + # instead of going through the full logical replication process. + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, payload integer)") + vanilla_pg.safe_psql( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120), testcolumn1 int, testcolumn2 int, testcolumn3 int);" + ) + connstr = endpoint.connstr().replace("'", "''") + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + + # Wait logical replication channel to be established + logical_replication_sync(vanilla_pg, endpoint) + vanilla_pg.stop() + endpoint.stop() + + env.pageserver.assert_log_contains("enabling aux file v2 support") + with env.pageserver.http_client() as client: + # aux file v2 flag should be enabled at this point + assert client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)["aux_file_v2"] + with env.pageserver.http_client() as client: + tenant_config = client.tenant_config(tenant_id).effective_config + tenant_config["try_enable_aux_file_v2"] = False + client.set_tenant_config(tenant_id, tenant_config) + # the flag should still be enabled + assert client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)["aux_file_v2"] + env.pageserver.restart() + with env.pageserver.http_client() as client: + # aux file v2 flag should be persisted + assert client.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id)["aux_file_v2"]