fix customize config test

Signed-off-by: Alex Chi Z <chi@neon.tech>

control_plane/src/pageserver.rs

@@ -571,6 +571,11 @@ impl PageServerNode {
                 .map(|x| x.parse::<bool>())
                 .transpose()
                 .context("Failed to parse 'basebackup_cache_enabled' as bool")?,
+            rel_size_v1_access_disabled: settings
+                .remove("rel_size_v1_access_disabled")
+                .map(|x| x.parse::<bool>())
+                .transpose()
+                .context("Failed to parse 'rel_size_v1_access_disabled' as bool")?,
         };
         if !settings.is_empty() {
             bail!("Unrecognized tenant settings: {settings:?}")

libs/pageserver_api/src/config.rs

@@ -651,6 +651,9 @@ pub struct TenantConfigToml {
     // FIXME: Remove skip_serializing_if when the feature is stable.
     #[serde(skip_serializing_if = "std::ops::Not::not")]
     pub basebackup_cache_enabled: bool,
+
+    #[serde(skip_serializing_if = "std::ops::Not::not")]
+    pub rel_size_v1_access_disabled: bool,
 }
 
 pub mod defaults {
@@ -959,6 +962,7 @@ impl Default for TenantConfigToml {
             sampling_ratio: None,
             relsize_snapshot_cache_capacity: DEFAULT_RELSIZE_SNAPSHOT_CACHE_CAPACITY,
             basebackup_cache_enabled: false,
+            rel_size_v1_access_disabled: false,
         }
     }
 }

libs/pageserver_api/src/key.rs

@@ -519,6 +519,15 @@ pub fn rel_dir_to_key(spcnode: Oid, dbnode: Oid) -> Key {
     }
 }
 
+pub const REL_DIR_MIGRATION_KEY: Key = Key {
+    field1: REL_DIR_KEY_PREFIX,
+    field2: 0,
+    field3: 0,
+    field4: 0,
+    field5: 0,
+    field6: 0,
+};
+
 #[inline(always)]
 pub fn rel_tag_sparse_key(spcnode: Oid, dbnode: Oid, relnode: Oid, forknum: u8) -> Key {
     Key {

libs/pageserver_api/src/models.rs

@@ -646,6 +646,8 @@ pub struct TenantConfigPatch {
     pub relsize_snapshot_cache_capacity: FieldPatch<usize>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub basebackup_cache_enabled: FieldPatch<bool>,
+    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
+    pub rel_size_v1_access_disabled: FieldPatch<bool>,
 }
 
 /// Like [`crate::config::TenantConfigToml`], but preserves the information
@@ -783,6 +785,9 @@ pub struct TenantConfig {
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub basebackup_cache_enabled: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rel_size_v1_access_disabled: Option<bool>,
 }
 
 impl TenantConfig {
@@ -830,6 +835,7 @@ impl TenantConfig {
             mut sampling_ratio,
             mut relsize_snapshot_cache_capacity,
             mut basebackup_cache_enabled,
+            mut rel_size_v1_access_disabled,
         } = self;
 
         patch.checkpoint_distance.apply(&mut checkpoint_distance);
@@ -939,6 +945,9 @@ impl TenantConfig {
         patch
             .basebackup_cache_enabled
             .apply(&mut basebackup_cache_enabled);
+        patch
+            .rel_size_v1_access_disabled
+            .apply(&mut rel_size_v1_access_disabled);
 
         Ok(Self {
             checkpoint_distance,
@@ -980,6 +989,7 @@ impl TenantConfig {
             sampling_ratio,
             relsize_snapshot_cache_capacity,
             basebackup_cache_enabled,
+            rel_size_v1_access_disabled,
         })
     }
 
@@ -1094,6 +1104,9 @@ impl TenantConfig {
             basebackup_cache_enabled: self
                 .basebackup_cache_enabled
                 .unwrap_or(global_conf.basebackup_cache_enabled),
+            rel_size_v1_access_disabled: self
+                .rel_size_v1_access_disabled
+                .unwrap_or(global_conf.rel_size_v1_access_disabled),
         }
     }
 }
@@ -1526,12 +1539,15 @@ pub struct OffloadedTimelineInfo {
     pub archived_at: chrono::DateTime<chrono::Utc>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+/// The state of the rel size migration. This is persisted in the DbDir key and index part. Do not change without considering
+/// compatibility.
+#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum RelSizeMigration {
     /// The tenant is using the old rel_size format.
     /// Note that this enum is persisted as `Option<RelSizeMigration>` in the index part, so
     /// `None` is the same as `Some(RelSizeMigration::Legacy)`.
+    #[default]
     Legacy,
     /// The tenant is migrating to the new rel_size format. Both old and new rel_size format are
     /// persisted in the storage. The read path will read both formats and validate them.

pageserver/src/http/routes.rs

@@ -484,7 +484,7 @@ async fn build_timeline_info_common(
         *timeline.get_applied_gc_cutoff_lsn(),
     );
 
-    let (rel_size_migration, rel_size_migrated_at) = timeline.get_rel_size_v2_status();
+    let (rel_size_migration, rel_size_migrated_at) = timeline.get_rel_size_v2_cached_status();
 
     let info = TimelineInfo {
         tenant_id: timeline.tenant_shard_id,

pageserver/src/pgdatadir_mapping.rs

@@ -16,10 +16,10 @@ use anyhow::Context;
 use bytes::{Buf, Bytes, BytesMut};
 use enum_map::Enum;
 use pageserver_api::key::{
-    AUX_FILES_KEY, CHECKPOINT_KEY, CONTROLFILE_KEY, CompactKey, DBDIR_KEY, Key, RelDirExists,
-    TWOPHASEDIR_KEY, dbdir_key_range, rel_block_to_key, rel_dir_to_key, rel_key_range,
-    rel_size_to_key, rel_tag_sparse_key, rel_tag_sparse_key_range, relmap_file_key,
-    repl_origin_key, repl_origin_key_range, slru_block_to_key, slru_dir_to_key,
+    AUX_FILES_KEY, CHECKPOINT_KEY, CONTROLFILE_KEY, CompactKey, DBDIR_KEY, Key,
+    REL_DIR_MIGRATION_KEY, RelDirExists, TWOPHASEDIR_KEY, dbdir_key_range, rel_block_to_key,
+    rel_dir_to_key, rel_key_range, rel_size_to_key, rel_tag_sparse_key, rel_tag_sparse_key_range,
+    relmap_file_key, repl_origin_key, repl_origin_key_range, slru_block_to_key, slru_dir_to_key,
     slru_segment_key_range, slru_segment_size_to_key, twophase_file_key, twophase_key_range,
 };
 use pageserver_api::keyspace::{KeySpaceRandomAccum, SparseKeySpace};
@@ -685,12 +685,16 @@ impl Timeline {
         // then check if the database was already initialized.
         // get_rel_exists can be called before dbdir is created.
         let buf = version.get(self, DBDIR_KEY, ctx).await?;
-        let dbdirs = DbDirectory::des(&buf)?.dbdirs;
+        let dbdir = DbDirectory::des(&buf)?;
+        let dbdirs = &dbdir.dbdirs;
         if !dbdirs.contains_key(&(tag.spcnode, tag.dbnode)) {
             return Ok(false);
         }
 
-        let (v2_status, migrated_lsn) = self.get_rel_size_v2_status();
+        let migration_history = version.sparse_get(self, REL_DIR_MIGRATION_KEY, ctx).await?;
+        let migration_history = RelDirMigrationHistory::from_bytes(migration_history)
+            .context("failed to deserialize rel dir migration history")?;
+        let v2_status = migration_history.status;
 
         match v2_status {
             RelSizeMigration::Legacy => {
@@ -699,15 +703,6 @@ impl Timeline {
                     .await?;
                 Ok(v1_exists)
             }
-            RelSizeMigration::Migrating | RelSizeMigration::Migrated
-                if version.get_lsn() < migrated_lsn.unwrap_or(Lsn(0)) =>
-            {
-                // For requests below the migrated LSN, we still use the v1 read path.
-                let v1_exists = self
-                    .get_rel_exists_in_reldir_v1(tag, version, deserialized_reldir_v1, ctx)
-                    .await?;
-                Ok(v1_exists)
-            }
             RelSizeMigration::Migrating => {
                 let v1_exists = self
                     .get_rel_exists_in_reldir_v1(tag, version, deserialized_reldir_v1, ctx)
@@ -720,9 +715,12 @@ impl Timeline {
                             "inconsistent v1/v2 reldir keyspace for rel {}: v1_exists={}, v2_exists={}",
                             tag,
                             v1_exists,
-                            v2_exists
+                            v2_exists,
                         );
                     }
+                    Err(e) if e.is_cancel() => {
+                        // Cancellation errors are fine to ignore, do not log.
+                    }
                     Err(e) => {
                         tracing::warn!("failed to get rel exists in v2: {e}");
                     }
@@ -780,6 +778,10 @@ impl Timeline {
             .await?;
         let mut rels = HashSet::new();
         for (key, val) in results {
+            if key == REL_DIR_MIGRATION_KEY {
+                // The key that determines the current migration status, skip it.
+                continue;
+            }
             let val = RelDirExists::decode(&val?).map_err(|_| {
                 PageReconstructError::Other(anyhow::anyhow!(
                     "invalid reldir key: decode failed, {}",
@@ -811,11 +813,11 @@ impl Timeline {
                 forknum: key.field5,
             };
             if val == RelDirExists::Removed {
-                debug_assert!(!rels.contains(&tag), "removed reltag in v2");
+                debug_assert!(!rels.contains(&tag), "removed reltag in v2: {tag}");
                 continue;
             }
             let did_not_contain = rels.insert(tag);
-            debug_assert!(did_not_contain, "duplicate reltag in v2");
+            debug_assert!(did_not_contain, "duplicate reltag in v2: {tag}");
         }
         Ok(rels)
     }
@@ -835,20 +837,16 @@ impl Timeline {
         version: Version<'_>,
         ctx: &RequestContext,
     ) -> Result<HashSet<RelTag>, PageReconstructError> {
-        let (v2_status, migrated_lsn) = self.get_rel_size_v2_status();
+        let reldir_migration_history = version.sparse_get(self, REL_DIR_MIGRATION_KEY, ctx).await?;
+        let reldir_migration_history = RelDirMigrationHistory::from_bytes(reldir_migration_history)
+            .context("failed to deserialize rel dir migration history")?;
+        let v2_status = reldir_migration_history.status;
 
         match v2_status {
             RelSizeMigration::Legacy => {
                 let rels_v1 = self.list_rels_v1(spcnode, dbnode, version, ctx).await?;
                 Ok(rels_v1)
             }
-            RelSizeMigration::Migrating | RelSizeMigration::Migrated
-                if version.get_lsn() < migrated_lsn.unwrap_or(Lsn(0)) =>
-            {
-                // For requests below the migrated LSN, we still use the v1 read path.
-                let rels_v1 = self.list_rels_v1(spcnode, dbnode, version, ctx).await?;
-                Ok(rels_v1)
-            }
             RelSizeMigration::Migrating => {
                 let rels_v1 = self.list_rels_v1(spcnode, dbnode, version, ctx).await?;
                 let rels_v2_res = self.list_rels_v2(spcnode, dbnode, version, ctx).await;
@@ -863,6 +861,9 @@ impl Timeline {
                             rels_v2.len()
                         );
                     }
+                    Err(e) if e.is_cancel() => {
+                        // Cancellation errors are fine to ignore, do not log.
+                    }
                     Err(e) => {
                         tracing::warn!("failed to list rels in v2: {e}");
                     }
@@ -1724,6 +1725,8 @@ pub struct RelDirMode {
     current_status: RelSizeMigration,
     // Whether we should initialize the v2 keyspace or not.
     initialize: bool,
+    // Whether we should disable v1 access starting this LSNor not
+    disable_v1: bool,
 }
 
 impl DatadirModification<'_> {
@@ -2085,44 +2088,71 @@ impl DatadirModification<'_> {
     ///
     /// As this function is only used on the write path, we do not need to read the migrated_at
     /// field.
-    pub fn maybe_enable_rel_size_v2(&mut self, is_create: bool) -> anyhow::Result<RelDirMode> {
+    pub(crate) fn maybe_enable_rel_size_v2(
+        &mut self,
+        migration_history: &RelDirMigrationHistory,
+        is_create: bool,
+    ) -> anyhow::Result<RelDirMode> {
         // TODO: define the behavior of the tenant-level config flag and use feature flag to enable this feature
+        let expected_status = self.tline.get_rel_size_v2_expected_state();
+        let persistent_status = migration_history.status.clone();
 
-        let (status, _) = self.tline.get_rel_size_v2_status();
-        let config = self.tline.get_rel_size_v2_enabled();
-        match (config, status) {
-            (false, RelSizeMigration::Legacy) => {
+        // Only initialize the v2 keyspace on new relation creation. No initialization
+        // during `timeline_create` (TODO: fix this, we should allow, but currently it
+        // hits consistency issues).
+        let can_update = is_create && !self.is_importing_pgdata;
+
+        match (expected_status, persistent_status) {
+            (RelSizeMigration::Legacy, RelSizeMigration::Legacy) => {
                 // tenant config didn't enable it and we didn't write any reldir_v2 key yet
                 Ok(RelDirMode {
                     current_status: RelSizeMigration::Legacy,
                     initialize: false,
+                    disable_v1: false,
                 })
             }
-            (false, status @ RelSizeMigration::Migrating | status @ RelSizeMigration::Migrated) => {
-                // index_part already persisted that the timeline has enabled rel_size_v2
+            (
+                RelSizeMigration::Legacy,
+                current_status @ RelSizeMigration::Migrating
+                | current_status @ RelSizeMigration::Migrated,
+            ) => {
+                // already persisted that the timeline has enabled rel_size_v2, cannot rollback
                 Ok(RelDirMode {
-                    current_status: status,
+                    current_status,
                     initialize: false,
+                    disable_v1: false,
                 })
             }
-            (true, RelSizeMigration::Legacy) => {
+            (
+                expected_status @ RelSizeMigration::Migrating
+                | expected_status @ RelSizeMigration::Migrated,
+                RelSizeMigration::Legacy,
+            ) => {
                 // The first time we enable it, we need to persist it in `index_part.json`
                 // The caller should update the reldir status once the initialization is done.
-                //
-                // Only initialize the v2 keyspace on new relation creation. No initialization
-                // during `timeline_create` (TODO: fix this, we should allow, but currently it
-                // hits consistency issues).
+
                 Ok(RelDirMode {
                     current_status: RelSizeMigration::Legacy,
-                    initialize: is_create && !self.is_importing_pgdata,
+                    initialize: can_update,
+                    disable_v1: can_update && expected_status == RelSizeMigration::Migrated,
                 })
             }
-            (true, status @ RelSizeMigration::Migrating | status @ RelSizeMigration::Migrated) => {
-                // index_part already persisted that the timeline has enabled rel_size_v2
-                // and we don't need to do anything
+            (RelSizeMigration::Migrating, current_status @ RelSizeMigration::Migrating)
+            | (RelSizeMigration::Migrated, current_status @ RelSizeMigration::Migrated)
+            | (RelSizeMigration::Migrating, current_status @ RelSizeMigration::Migrated) => {
+                // Keep the current state
                 Ok(RelDirMode {
-                    current_status: status,
+                    current_status,
                     initialize: false,
+                    disable_v1: false,
+                })
+            }
+            (RelSizeMigration::Migrated, RelSizeMigration::Migrating) => {
+                // Switch to v2-only mode
+                Ok(RelDirMode {
+                    current_status: RelSizeMigration::Migrating,
+                    initialize: false,
+                    disable_v1: can_update,
                 })
             }
         }
@@ -2136,14 +2166,19 @@ impl DatadirModification<'_> {
         img: Bytes,
         ctx: &RequestContext,
     ) -> Result<(), WalIngestError> {
-        let v2_mode = self
-            .maybe_enable_rel_size_v2(false)
-            .map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
-
         // Add it to the directory (if it doesn't exist already)
         let buf = self.get(DBDIR_KEY, ctx).await?;
         let mut dbdir = DbDirectory::des(&buf)?;
 
+        let reldir_migration_history = self.sparse_get(REL_DIR_MIGRATION_KEY, ctx).await?;
+        let reldir_migration_history = RelDirMigrationHistory::from_bytes(reldir_migration_history)
+            .context("failed to deserialize rel dir migration history")
+            .map_err(WalIngestErrorKind::RelSizeV2Error)?;
+
+        let v2_mode = self
+            .maybe_enable_rel_size_v2(&reldir_migration_history, false)
+            .map_err(WalIngestErrorKind::RelSizeV2Error)?;
+
         let r = dbdir.dbdirs.insert((spcnode, dbnode), true);
         if r.is_none() || r == Some(false) {
             // The dbdir entry didn't exist, or it contained a
@@ -2290,15 +2325,6 @@ impl DatadirModification<'_> {
                     sparse_rel_dir_key,
                     Value::Image(RelDirExists::Exists.encode()),
                 );
-                tracing::info!(
-                    "migrated rel_size_v2: {}",
-                    RelTag {
-                        spcnode,
-                        dbnode,
-                        relnode,
-                        forknum
-                    }
-                );
                 rel_cnt += 1;
             }
         }
@@ -2307,9 +2333,6 @@ impl DatadirModification<'_> {
             self.lsn,
             rel_cnt
         );
-        self.tline
-            .update_rel_size_v2_status(RelSizeMigration::Migrating, Some(self.lsn))
-            .map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
         Ok::<_, WalIngestError>(())
     }
 
@@ -2392,25 +2415,32 @@ impl DatadirModification<'_> {
         // It's possible that this is the first rel for this db in this
         // tablespace.  Create the reldir entry for it if so.
         let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await?)?;
+        let reldir_migration_history = self.sparse_get(REL_DIR_MIGRATION_KEY, ctx).await?;
+        let mut reldir_migration_history =
+            RelDirMigrationHistory::from_bytes(reldir_migration_history)
+                .context("failed to deserialize rel dir migration history")
+                .map_err(WalIngestErrorKind::RelSizeV2Error)?;
+
+        let mut is_dbdir_dirty = false;
+        let mut is_reldirv2_status_dirty = false;
 
         let dbdir_exists =
             if let hash_map::Entry::Vacant(e) = dbdir.dbdirs.entry((rel.spcnode, rel.dbnode)) {
                 // Didn't exist. Update dbdir
                 e.insert(false);
-                let buf = DbDirectory::ser(&dbdir)?;
                 self.pending_directory_entries.push((
                     DirectoryKind::Db,
                     MetricsUpdate::Set(dbdir.dbdirs.len() as u64),
                 ));
-                self.put(DBDIR_KEY, Value::Image(buf.into()));
+                is_dbdir_dirty = true;
                 false
             } else {
                 true
             };
 
         let mut v2_mode = self
-            .maybe_enable_rel_size_v2(true)
-            .map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
+            .maybe_enable_rel_size_v2(&reldir_migration_history, true)
+            .map_err(WalIngestErrorKind::RelSizeV2Error)?;
 
         if v2_mode.initialize {
             if let Err(e) = self.initialize_rel_size_v2_keyspace(ctx, &dbdir).await {
@@ -2418,8 +2448,40 @@ impl DatadirModification<'_> {
                 // TODO: circuit breaker so that it won't retry forever
             } else {
                 v2_mode.current_status = RelSizeMigration::Migrating;
+                reldir_migration_history.status = RelSizeMigration::Migrating;
+                reldir_migration_history.v2_enabled_at = Some(self.lsn);
+                is_reldirv2_status_dirty = true;
             }
         }
+        if v2_mode.disable_v1 {
+            v2_mode.current_status = RelSizeMigration::Migrated;
+            reldir_migration_history.status = RelSizeMigration::Migrated;
+            reldir_migration_history.v1_disabled_at = Some(self.lsn);
+            is_reldirv2_status_dirty = true;
+        }
+
+        if is_dbdir_dirty {
+            let buf = DbDirectory::ser(&dbdir)?;
+            self.put(DBDIR_KEY, Value::Image(buf.into()));
+        }
+
+        if is_reldirv2_status_dirty {
+            self.tline
+                .update_rel_size_v2_status(
+                    reldir_migration_history.status.clone(),
+                    reldir_migration_history.v2_enabled_at,
+                )
+                .map_err(WalIngestErrorKind::RelSizeV2Error)?;
+            self.put(
+                REL_DIR_MIGRATION_KEY,
+                Value::Image(
+                    reldir_migration_history
+                        .encode()
+                        .context("failed to serialize rel dir migration history")
+                        .map_err(WalIngestErrorKind::RelSizeV2Error)?,
+                ),
+            );
+        }
 
         if v2_mode.current_status != RelSizeMigration::Migrated {
             self.put_rel_creation_v1(rel, dbdir_exists, ctx).await?;
@@ -2580,9 +2642,13 @@ impl DatadirModification<'_> {
         drop_relations: HashMap<(u32, u32), Vec<RelTag>>,
         ctx: &RequestContext,
     ) -> Result<(), WalIngestError> {
+        let reldir_migration_history = self.sparse_get(REL_DIR_MIGRATION_KEY, ctx).await?;
+        let reldir_migration_history = RelDirMigrationHistory::from_bytes(reldir_migration_history)
+            .context("failed to deserialize rel dir migration history")
+            .map_err(WalIngestErrorKind::RelSizeV2Error)?;
         let v2_mode = self
-            .maybe_enable_rel_size_v2(false)
-            .map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
+            .maybe_enable_rel_size_v2(&reldir_migration_history, false)
+            .map_err(WalIngestErrorKind::RelSizeV2Error)?;
         match v2_mode.current_status {
             RelSizeMigration::Legacy => {
                 self.put_rel_drop_v1(drop_relations, ctx).await?;
@@ -2600,6 +2666,12 @@ impl DatadirModification<'_> {
                             );
                         }
                     }
+                    Err(WalIngestError {
+                        kind: WalIngestErrorKind::Cancelled,
+                        ..
+                    }) => {
+                        // Cancellation errors are fine to ignore, do not log.
+                    }
                     Err(e) => {
                         tracing::warn!("error dropping rels: {}", e);
                     }
@@ -3149,6 +3221,32 @@ impl Version<'_> {
 
 //--- Metadata structs stored in key-value pairs in the repository.
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub(crate) struct RelDirMigrationHistory {
+    pub(crate) status: RelSizeMigration,
+    pub(crate) v2_enabled_at: Option<Lsn>,
+    pub(crate) v1_disabled_at: Option<Lsn>,
+}
+
+impl RelDirMigrationHistory {
+    pub(crate) fn from_bytes(bytes: Option<Bytes>) -> Result<Self, serde_json::Error> {
+        match bytes {
+            Some(bytes) => {
+                if bytes.is_empty() {
+                    return Ok(Self::default());
+                }
+                let history = serde_json::from_slice(&bytes)?;
+                Ok(history)
+            }
+            None => Ok(Self::default()),
+        }
+    }
+
+    pub(crate) fn encode(&self) -> Result<Bytes, serde_json::Error> {
+        serde_json::to_vec(self).map(Bytes::from)
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct DbDirectory {
     // (spcnode, dbnode) -> (do relmapper and PG_VERSION files exist)

pageserver/src/tenant.rs

@@ -5090,7 +5090,8 @@ impl TenantShard {
             src_timeline.pg_version,
         );
 
-        let (rel_size_v2_status, rel_size_migrated_at) = src_timeline.get_rel_size_v2_status();
+        let (rel_size_v2_status, rel_size_migrated_at) =
+            src_timeline.get_rel_size_v2_cached_status();
         let (uninitialized_timeline, _timeline_ctx) = self
             .prepare_new_timeline(
                 dst_id,

pageserver/src/tenant/remote_timeline_client/index.rs

@@ -83,6 +83,7 @@ pub struct IndexPart {
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
 
+    /// Deprecated: the field is not used anymore and the source of truth is now stored in the dbdir key.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub(crate) rel_size_migration: Option<RelSizeMigration>,
 
@@ -115,8 +116,7 @@ pub struct IndexPart {
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub(crate) marked_invisible_at: Option<NaiveDateTime>,
 
-    /// The LSN at which we started the rel size migration. Accesses below this LSN should be
-    /// processed with the v1 read path. Usually this LSN should be set together with `rel_size_migration`.
+    /// Deprecated: the field is not used anymore and the source of truth is now stored in the dbdir key.
     #[serde(skip_serializing_if = "Option::is_none", default)]
     pub(crate) rel_size_migrated_at: Option<Lsn>,
 }

pageserver/src/tenant/timeline.rs

@@ -441,7 +441,7 @@ pub struct Timeline {
     /// heatmap on demand.
     heatmap_layers_downloader: Mutex<Option<heatmap_layers_downloader::HeatmapLayersDownloader>>,
 
-    pub(crate) rel_size_v2_status: ArcSwap<(Option<RelSizeMigration>, Option<Lsn>)>,
+    pub(crate) rel_size_v2_cached_status: ArcSwap<(Option<RelSizeMigration>, Option<Lsn>)>,
 
     wait_lsn_log_slow: tokio::sync::Semaphore,
 
@@ -2883,19 +2883,33 @@ impl Timeline {
             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
     }
 
-    /// Returns `true` if the rel_size_v2 config is enabled. NOTE: the write path and read path
-    /// should look at `get_rel_size_v2_status()` to get the actual status of the timeline. It is
-    /// possible that the index part persists the state while the config doesn't get persisted.
-    pub(crate) fn get_rel_size_v2_enabled(&self) -> bool {
+    /// Returns the expected state of the rel size migration. The actual state is persisted in the
+    /// DbDir key.
+    ///
+    /// The expected state is the state that the tenant config expects.
+    pub(crate) fn get_rel_size_v2_expected_state(&self) -> RelSizeMigration {
         let tenant_conf = self.tenant_conf.load();
-        tenant_conf
+        let v2_enabled = tenant_conf
             .tenant_conf
             .rel_size_v2_enabled
-            .unwrap_or(self.conf.default_tenant_conf.rel_size_v2_enabled)
+            .unwrap_or(self.conf.default_tenant_conf.rel_size_v2_enabled);
+        let v1_access_disabled = tenant_conf
+            .tenant_conf
+            .rel_size_v1_access_disabled
+            .unwrap_or(self.conf.default_tenant_conf.rel_size_v1_access_disabled);
+
+        match (v2_enabled, v1_access_disabled) {
+            (true, false) => RelSizeMigration::Migrating,
+            (true, true) => RelSizeMigration::Migrated,
+            (false, true) => RelSizeMigration::Legacy, // This should never happen
+            (false, false) => RelSizeMigration::Legacy,
+        }
     }
 
-    pub(crate) fn get_rel_size_v2_status(&self) -> (RelSizeMigration, Option<Lsn>) {
-        let (status, migrated_at) = self.rel_size_v2_status.load().as_ref().clone();
+    /// DO NOT use this API in the read/write path to determine the rel size migration status. The source of truth is the dbdir key.
+    /// This API is only used for the timeline info struct to get the latest cached status.
+    pub(crate) fn get_rel_size_v2_cached_status(&self) -> (RelSizeMigration, Option<Lsn>) {
+        let (status, migrated_at) = self.rel_size_v2_cached_status.load().as_ref().clone();
         (status.unwrap_or(RelSizeMigration::Legacy), migrated_at)
     }
 
@@ -3336,7 +3350,7 @@ impl Timeline {
 
                 heatmap_layers_downloader: Mutex::new(None),
 
-                rel_size_v2_status: ArcSwap::from_pointee((
+                rel_size_v2_cached_status: ArcSwap::from_pointee((
                     rel_size_v2_status,
                     rel_size_migrated_at,
                 )),
@@ -3429,10 +3443,11 @@ impl Timeline {
         rel_size_v2_status: RelSizeMigration,
         rel_size_migrated_at: Option<Lsn>,
     ) -> anyhow::Result<()> {
-        self.rel_size_v2_status.store(Arc::new((
+        self.rel_size_v2_cached_status.store(Arc::new((
             Some(rel_size_v2_status.clone()),
             rel_size_migrated_at,
         )));
+        // The index_part upload is not used as source of truth anymore, but we still need to upload it to make it work across branches.
         self.remote_client
             .schedule_index_upload_for_rel_size_v2_status_update(
                 rel_size_v2_status,

pageserver/src/walingest.rs

@@ -135,7 +135,7 @@ pub enum WalIngestErrorKind {
     #[error(transparent)]
     EncodeAuxFileError(anyhow::Error),
     #[error(transparent)]
-    MaybeRelSizeV2Error(anyhow::Error),
+    RelSizeV2Error(anyhow::Error),
 
     #[error("timeline shutting down")]
     Cancelled,

test_runner/fixtures/neon_fixtures.py

@@ -1312,8 +1312,8 @@ class NeonEnv:
                 )
 
             tenant_config = ps_cfg.setdefault("tenant_config", {})
-            # This feature is pending rollout.
-            # tenant_config["rel_size_v2_enabled"] = True
+            # Enable relsize_v2 by default in tests.
+            tenant_config["rel_size_v2_enabled"] = True
 
             # Test authors tend to forget about the default 10min initial lease deadline
             # when writing tests, which turns their immediate gc requests via mgmt API

test_runner/performance/test_perf_many_relations.py

@@ -80,7 +80,12 @@ def test_perf_simple_many_relations_reldir(
     """
     Test creating many relations in a single database.
     """
-    env = neon_env_builder.init_start(initial_tenant_conf={"rel_size_v2_enabled": reldir != "v1"})
+    env = neon_env_builder.init_start(
+        initial_tenant_conf={
+            "rel_size_v2_enabled": reldir != "v1",
+            "rel_size_v1_access_disabled": reldir == "v2",
+        }
+    )
     ep = env.endpoints.create_start(
         "main",
         config_lines=[
@@ -108,10 +113,6 @@ def test_perf_simple_many_relations_reldir(
             == "migrating"
         )
     elif reldir == "v2":
-        # only read/write to the v2 keyspace
-        env.pageserver.http_client().timeline_patch_index_part(
-            env.initial_tenant, env.initial_timeline, {"rel_size_migration": "migrated"}
-        )
         assert (
             env.pageserver.http_client().timeline_detail(env.initial_tenant, env.initial_timeline)[
                 "rel_size_migration"

test_runner/regress/test_attach_tenant_config.py

@@ -183,7 +183,7 @@ def test_fully_custom_config(positive_env: NeonEnv):
         "lsn_lease_length": "1m",
         "lsn_lease_length_for_ts": "5s",
         "timeline_offloading": False,
-        "rel_size_v2_enabled": True,
+        "rel_size_v2_enabled": False,
         "relsize_snapshot_cache_capacity": 10000,
         "gc_compaction_enabled": False,
         "gc_compaction_verification": False,
@@ -194,6 +194,7 @@ def test_fully_custom_config(positive_env: NeonEnv):
             "numerator": 0,
             "denominator": 10,
         },
+        "rel_size_v1_access_disabled": True,
     }
 
     vps_http = env.storage_controller.pageserver_api()

test_runner/regress/test_compatibility.py

@@ -581,12 +581,10 @@ def test_historic_storage_formats(
         # This dataset was created at a time where we decided to migrate to v2 reldir by simply disabling writes to v1
         # and starting writing to v2. This was too risky and we have reworked the migration plan. Therefore, we should
         # opt in full relv2 mode for this dataset.
-        for timeline in timelines:
-            env.pageserver.http_client().timeline_patch_index_part(
-                dataset.tenant_id,
-                timeline["timeline_id"],
-                {"force_index_update": True, "rel_size_migration": "migrated"},
-            )
+        env.pageserver.http_client().patch_tenant_config(
+            dataset.tenant_id,
+            {"rel_size_v1_access_disabled": True, "rel_size_v2_enabled": True},
+        )
 
     # Import tenant does not create the timeline on safekeepers,
     # because it is a debug handler and the timeline may have already been

test_runner/regress/test_pg_regress.py

@@ -123,9 +123,10 @@ def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: End
 def patch_tenant_conf(tenant_conf: dict[str, Any], reldir_type: str) -> dict[str, Any]:
     tenant_conf = tenant_conf.copy()
     if reldir_type == "v2":
-        tenant_conf["rel_size_v2_enabled"] = "true"
+        tenant_conf["rel_size_v2_enabled"] = True
+        tenant_conf["rel_size_v1_access_disabled"] = True
     else:
-        tenant_conf["rel_size_v2_enabled"] = "false"
+        tenant_conf["rel_size_v2_enabled"] = False
     return tenant_conf