rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 05:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pageserver): add reldir migration configs (#10439)

Browse Source 
## Problem

Part of #9516 per RFC at https://github.com/neondatabase/neon/pull/10412

## Summary of changes

Adding the necessary config items and index_part items for the large
relation count work.

---------

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-01-20 15:44:12 -05:00
committed by GitHub
parent e781cf6dd8
commit 2de2b26c62
7 changed files with 138 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
control_plane/src/pageserver.rs
View File

@@ -418,6 +418,11 @@ impl PageServerNode {
.map(serde_json::from_str)
.transpose()
.context("parse `wal_receiver_protocol_override` from json")?,
rel_size_v2_enabled: settings
.remove("rel_size_v2_enabled")
.map(|x| x.parse::<bool>())
.transpose()
.context("Failed to parse 'rel_size_v2_enabled' as bool")?,
};
if !settings.is_empty() {
bail!("Unrecognized tenant settings: {settings:?}")

5
libs/pageserver_api/src/config.rs
View File

@@ -301,6 +301,10 @@ pub struct TenantConfigToml {
pub timeline_offloading: bool,
pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
/// Enable rel_size_v2 for this tenant. Once enabled, the tenant will persist this information into
/// `index_part.json`, and it cannot be reversed.
pub rel_size_v2_enabled: Option<bool>,
}
pub mod defaults {
@@ -538,6 +542,7 @@ impl Default for TenantConfigToml {
lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS,
timeline_offloading: false,
wal_receiver_protocol_override: None,
rel_size_v2_enabled: None,
}
}
}

6
libs/pageserver_api/src/models.rs
View File

@@ -497,6 +497,8 @@ pub struct TenantConfigPatch {
pub timeline_offloading: FieldPatch<bool>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub wal_receiver_protocol_override: FieldPatch<PostgresClientProtocol>,
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
pub rel_size_v2_enabled: FieldPatch<bool>,
}
/// An alternative representation of `pageserver::tenant::TenantConf` with
@@ -528,6 +530,7 @@ pub struct TenantConfig {
pub lsn_lease_length_for_ts: Option<String>,
pub timeline_offloading: Option<bool>,
pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
pub rel_size_v2_enabled: Option<bool>,
}
impl TenantConfig {
@@ -557,6 +560,7 @@ impl TenantConfig {
mut lsn_lease_length_for_ts,
mut timeline_offloading,
mut wal_receiver_protocol_override,
mut rel_size_v2_enabled,
} = self;
patch.checkpoint_distance.apply(&mut checkpoint_distance);
@@ -601,6 +605,7 @@ impl TenantConfig {
patch
.wal_receiver_protocol_override
.apply(&mut wal_receiver_protocol_override);
patch.rel_size_v2_enabled.apply(&mut rel_size_v2_enabled);
Self {
checkpoint_distance,
@@ -627,6 +632,7 @@ impl TenantConfig {
lsn_lease_length_for_ts,
timeline_offloading,
wal_receiver_protocol_override,
rel_size_v2_enabled,
}
}
}

1
pageserver/src/tenant.rs
View File

@@ -5475,6 +5475,7 @@ pub(crate) mod harness {
lsn_lease_length_for_ts: Some(tenant_conf.lsn_lease_length_for_ts),
timeline_offloading: Some(tenant_conf.timeline_offloading),
wal_receiver_protocol_override: tenant_conf.wal_receiver_protocol_override,
rel_size_v2_enabled: tenant_conf.rel_size_v2_enabled,
}
}
}

8
pageserver/src/tenant/config.rs
View File

@@ -357,6 +357,9 @@ pub struct TenantConfOpt {
#[serde(skip_serializing_if = "Option::is_none")]
pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rel_size_v2_enabled: Option<bool>,
}
impl TenantConfOpt {
@@ -425,6 +428,7 @@ impl TenantConfOpt {
wal_receiver_protocol_override: self
.wal_receiver_protocol_override
.or(global_conf.wal_receiver_protocol_override),
rel_size_v2_enabled: self.rel_size_v2_enabled.or(global_conf.rel_size_v2_enabled),
}
}
@@ -454,6 +458,7 @@ impl TenantConfOpt {
mut lsn_lease_length_for_ts,
mut timeline_offloading,
mut wal_receiver_protocol_override,
mut rel_size_v2_enabled,
} = self;
patch.checkpoint_distance.apply(&mut checkpoint_distance);
@@ -522,6 +527,7 @@ impl TenantConfOpt {
patch
.wal_receiver_protocol_override
.apply(&mut wal_receiver_protocol_override);
patch.rel_size_v2_enabled.apply(&mut rel_size_v2_enabled);
Ok(Self {
checkpoint_distance,
@@ -548,6 +554,7 @@ impl TenantConfOpt {
lsn_lease_length_for_ts,
timeline_offloading,
wal_receiver_protocol_override,
rel_size_v2_enabled,
})
}
}
@@ -603,6 +610,7 @@ impl From<TenantConfOpt> for models::TenantConfig {
lsn_lease_length_for_ts: value.lsn_lease_length_for_ts.map(humantime),
timeline_offloading: value.timeline_offloading,
wal_receiver_protocol_override: value.wal_receiver_protocol_override,
rel_size_v2_enabled: value.rel_size_v2_enabled,
}
}
}

115
pageserver/src/tenant/remote_timeline_client/index.rs
View File

@@ -79,6 +79,24 @@ pub struct IndexPart {
/// when this flag is introduced.
#[serde(skip_serializing_if = "Option::is_none", default)]
pub(crate) last_aux_file_policy: Option<AuxFilePolicy>,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub(crate) rel_size_migration: Option<RelSizeMigration>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum RelSizeMigration {
/// The tenant is using the old rel_size format.
/// Note that this enum is persisted as `Option<RelSizeMigration>` in the index part, so
/// `None` is the same as `Some(RelSizeMigration::Legacy)`.
Legacy,
/// The tenant is migrating to the new rel_size format. Both old and new rel_size format are
/// persisted in the index part. The read path will read both formats and merge them.
Migrating,
/// The tenant has migrated to the new rel_size format. Only the new rel_size format is persisted
/// in the index part, and the read path will not read the old format.
Migrated,
}
impl IndexPart {
@@ -97,10 +115,11 @@ impl IndexPart {
/// - 8: added `archived_at`
/// - 9: +gc_blocking
/// - 10: +import_pgdata
const LATEST_VERSION: usize = 10;
/// - 11: +rel_size_migration
const LATEST_VERSION: usize = 11;
// Versions we may see when reading from a bucket.
pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
pub const FILE_NAME: &'static str = "index_part.json";
@@ -116,6 +135,7 @@ impl IndexPart {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
}
}
@@ -416,6 +436,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -461,6 +482,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -507,6 +529,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -556,6 +579,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
@@ -600,6 +624,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -647,6 +672,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -699,6 +725,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: Some(AuxFilePolicy::V2),
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -756,6 +783,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: Default::default(),
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -814,6 +842,7 @@ mod tests {
gc_blocking: None,
last_aux_file_policy: Default::default(),
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -877,6 +906,7 @@ mod tests {
last_aux_file_policy: Default::default(),
archived_at: None,
import_pgdata: None,
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -952,7 +982,86 @@ mod tests {
started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
})))
}))),
rel_size_migration: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
assert_eq!(part, expected);
}
#[test]
fn v11_rel_size_migration_is_parsed() {
let example = r#"{
"version": 11,
"layer_metadata":{
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
},
"disk_consistent_lsn":"0/16960E8",
"metadata": {
"disk_consistent_lsn": "0/16960E8",
"prev_record_lsn": "0/1696070",
"ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
"ancestor_lsn": "0/0",
"latest_gc_cutoff_lsn": "0/1696070",
"initdb_lsn": "0/1696070",
"pg_version": 14
},
"gc_blocking": {
"started_at": "2024-07-19T09:00:00.123",
"reasons": ["DetachAncestor"]
},
"import_pgdata": {
"V1": {
"Done": {
"idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
"started_at": "2024-11-13T09:23:42.123",
"finished_at": "2024-11-13T09:42:23.123"
}
}
},
"rel_size_migration": "legacy"
}"#;
let expected = IndexPart {
version: 11,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
generation: Generation::none(),
shard: ShardIndex::unsharded()
}),
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
file_size: 9007199254741001,
generation: Generation::none(),
shard: ShardIndex::unsharded()
})
]),
disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
metadata: TimelineMetadata::new(
Lsn::from_str("0/16960E8").unwrap(),
Some(Lsn::from_str("0/1696070").unwrap()),
Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
Lsn::INVALID,
Lsn::from_str("0/1696070").unwrap(),
Lsn::from_str("0/1696070").unwrap(),
14,
).with_recalculated_checksum().unwrap(),
deleted_at: None,
lineage: Default::default(),
gc_blocking: Some(GcBlocking {
started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
}),
last_aux_file_policy: Default::default(),
archived_at: None,
import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
}))),
rel_size_migration: Some(RelSizeMigration::Legacy),
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();

1
test_runner/regress/test_attach_tenant_config.py
View File

@@ -176,6 +176,7 @@ def test_fully_custom_config(positive_env: NeonEnv):
"type": "interpreted",
"args": {"format": "bincode", "compression": {"zstd": {"level": 1}}},
},
"rel_size_v2_enabled": True,
}
vps_http = env.storage_controller.pageserver_api()