mirror of
https://github.com/neondatabase/neon.git
synced 2026-07-04 12:40:37 +00:00
Compare commits
7 Commits
communicat
...
skyzh/try-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5713ff31fc | ||
|
|
6e0682e9ad | ||
|
|
fd984a2750 | ||
|
|
00091a4fb1 | ||
|
|
d37cd4b282 | ||
|
|
7442d47c3e | ||
|
|
f6b1891564 |
@@ -571,6 +571,11 @@ impl PageServerNode {
|
||||
.map(|x| x.parse::<bool>())
|
||||
.transpose()
|
||||
.context("Failed to parse 'basebackup_cache_enabled' as bool")?,
|
||||
rel_size_v1_access_disabled: settings
|
||||
.remove("rel_size_v1_access_disabled")
|
||||
.map(|x| x.parse::<bool>())
|
||||
.transpose()
|
||||
.context("Failed to parse 'rel_size_v1_access_disabled' as bool")?,
|
||||
};
|
||||
if !settings.is_empty() {
|
||||
bail!("Unrecognized tenant settings: {settings:?}")
|
||||
|
||||
@@ -651,6 +651,9 @@ pub struct TenantConfigToml {
|
||||
// FIXME: Remove skip_serializing_if when the feature is stable.
|
||||
#[serde(skip_serializing_if = "std::ops::Not::not")]
|
||||
pub basebackup_cache_enabled: bool,
|
||||
|
||||
#[serde(skip_serializing_if = "std::ops::Not::not")]
|
||||
pub rel_size_v1_access_disabled: bool,
|
||||
}
|
||||
|
||||
pub mod defaults {
|
||||
@@ -959,6 +962,7 @@ impl Default for TenantConfigToml {
|
||||
sampling_ratio: None,
|
||||
relsize_snapshot_cache_capacity: DEFAULT_RELSIZE_SNAPSHOT_CACHE_CAPACITY,
|
||||
basebackup_cache_enabled: false,
|
||||
rel_size_v1_access_disabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -646,6 +646,8 @@ pub struct TenantConfigPatch {
|
||||
pub relsize_snapshot_cache_capacity: FieldPatch<usize>,
|
||||
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
|
||||
pub basebackup_cache_enabled: FieldPatch<bool>,
|
||||
#[serde(skip_serializing_if = "FieldPatch::is_noop")]
|
||||
pub rel_size_v1_access_disabled: FieldPatch<bool>,
|
||||
}
|
||||
|
||||
/// Like [`crate::config::TenantConfigToml`], but preserves the information
|
||||
@@ -783,6 +785,9 @@ pub struct TenantConfig {
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub basebackup_cache_enabled: Option<bool>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rel_size_v1_access_disabled: Option<bool>,
|
||||
}
|
||||
|
||||
impl TenantConfig {
|
||||
@@ -830,6 +835,7 @@ impl TenantConfig {
|
||||
mut sampling_ratio,
|
||||
mut relsize_snapshot_cache_capacity,
|
||||
mut basebackup_cache_enabled,
|
||||
mut rel_size_v1_access_disabled,
|
||||
} = self;
|
||||
|
||||
patch.checkpoint_distance.apply(&mut checkpoint_distance);
|
||||
@@ -939,6 +945,9 @@ impl TenantConfig {
|
||||
patch
|
||||
.basebackup_cache_enabled
|
||||
.apply(&mut basebackup_cache_enabled);
|
||||
patch
|
||||
.rel_size_v1_access_disabled
|
||||
.apply(&mut rel_size_v1_access_disabled);
|
||||
|
||||
Ok(Self {
|
||||
checkpoint_distance,
|
||||
@@ -980,6 +989,7 @@ impl TenantConfig {
|
||||
sampling_ratio,
|
||||
relsize_snapshot_cache_capacity,
|
||||
basebackup_cache_enabled,
|
||||
rel_size_v1_access_disabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1094,6 +1104,9 @@ impl TenantConfig {
|
||||
basebackup_cache_enabled: self
|
||||
.basebackup_cache_enabled
|
||||
.unwrap_or(global_conf.basebackup_cache_enabled),
|
||||
rel_size_v1_access_disabled: self
|
||||
.rel_size_v1_access_disabled
|
||||
.unwrap_or(global_conf.rel_size_v1_access_disabled),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1526,12 +1539,15 @@ pub struct OffloadedTimelineInfo {
|
||||
pub archived_at: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
/// The state of the rel size migration. This is persisted in the index part.
|
||||
/// compatibility.
|
||||
#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum RelSizeMigration {
|
||||
/// The tenant is using the old rel_size format.
|
||||
/// Note that this enum is persisted as `Option<RelSizeMigration>` in the index part, so
|
||||
/// `None` is the same as `Some(RelSizeMigration::Legacy)`.
|
||||
#[default]
|
||||
Legacy,
|
||||
/// The tenant is migrating to the new rel_size format. Both old and new rel_size format are
|
||||
/// persisted in the storage. The read path will read both formats and validate them.
|
||||
|
||||
@@ -518,6 +518,7 @@ impl Timeline {
|
||||
}
|
||||
|
||||
// Pre-deserialize the rel directory to avoid duplicated work in `get_relsize_cached`.
|
||||
// TODO: refactor this read path to use reldir v2.
|
||||
let reldir_key = rel_dir_to_key(spcnode, dbnode);
|
||||
let buf = version.get(self, reldir_key, ctx).await?;
|
||||
let reldir = RelDirectory::des(&buf)?;
|
||||
@@ -720,9 +721,12 @@ impl Timeline {
|
||||
"inconsistent v1/v2 reldir keyspace for rel {}: v1_exists={}, v2_exists={}",
|
||||
tag,
|
||||
v1_exists,
|
||||
v2_exists
|
||||
v2_exists,
|
||||
);
|
||||
}
|
||||
Err(e) if e.is_cancel() => {
|
||||
// Cancellation errors are fine to ignore, do not log.
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to get rel exists in v2: {e}");
|
||||
}
|
||||
@@ -811,11 +815,11 @@ impl Timeline {
|
||||
forknum: key.field5,
|
||||
};
|
||||
if val == RelDirExists::Removed {
|
||||
debug_assert!(!rels.contains(&tag), "removed reltag in v2");
|
||||
debug_assert!(!rels.contains(&tag), "removed reltag in v2: {tag}");
|
||||
continue;
|
||||
}
|
||||
let did_not_contain = rels.insert(tag);
|
||||
debug_assert!(did_not_contain, "duplicate reltag in v2");
|
||||
debug_assert!(did_not_contain, "duplicate reltag in v2: {tag}");
|
||||
}
|
||||
Ok(rels)
|
||||
}
|
||||
@@ -863,6 +867,9 @@ impl Timeline {
|
||||
rels_v2.len()
|
||||
);
|
||||
}
|
||||
Err(e) if e.is_cancel() => {
|
||||
// Cancellation errors are fine to ignore, do not log.
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to list rels in v2: {e}");
|
||||
}
|
||||
@@ -1724,6 +1731,8 @@ pub struct RelDirMode {
|
||||
current_status: RelSizeMigration,
|
||||
// Whether we should initialize the v2 keyspace or not.
|
||||
initialize: bool,
|
||||
// Whether we should disable v1 access starting this LSNor not
|
||||
disable_v1: bool,
|
||||
}
|
||||
|
||||
impl DatadirModification<'_> {
|
||||
@@ -2085,44 +2094,82 @@ impl DatadirModification<'_> {
|
||||
///
|
||||
/// As this function is only used on the write path, we do not need to read the migrated_at
|
||||
/// field.
|
||||
pub fn maybe_enable_rel_size_v2(&mut self, is_create: bool) -> anyhow::Result<RelDirMode> {
|
||||
pub(crate) fn maybe_enable_rel_size_v2(
|
||||
&mut self,
|
||||
lsn: Lsn,
|
||||
is_create: bool,
|
||||
) -> anyhow::Result<RelDirMode> {
|
||||
// TODO: define the behavior of the tenant-level config flag and use feature flag to enable this feature
|
||||
let expected_status = self.tline.get_rel_size_v2_expected_state();
|
||||
let (persistent_status, migrated_at) = self.tline.get_rel_size_v2_status();
|
||||
|
||||
let (status, _) = self.tline.get_rel_size_v2_status();
|
||||
let config = self.tline.get_rel_size_v2_enabled();
|
||||
match (config, status) {
|
||||
(false, RelSizeMigration::Legacy) => {
|
||||
// migrated_at LSN means the LSN where we "enable_v2" (but not "disable_v1")
|
||||
if let Some(migrated_at) = migrated_at
|
||||
&& lsn <= migrated_at
|
||||
&& persistent_status == RelSizeMigration::Migrating
|
||||
{
|
||||
// Revert the status to legacy if the write head LSN is before the migrating LSN.
|
||||
self.tline
|
||||
.update_rel_size_v2_status(RelSizeMigration::Legacy, None)?;
|
||||
}
|
||||
// TODO: what if the migration is at "migrated" state but we need to revert?
|
||||
|
||||
// Only initialize the v2 keyspace on new relation creation. No initialization
|
||||
// during `timeline_create` (TODO: fix this, we should allow, but currently it
|
||||
// hits expected consistency issues; need to ignore warnings).
|
||||
let can_update = is_create && !self.is_importing_pgdata;
|
||||
|
||||
match (expected_status, persistent_status) {
|
||||
(RelSizeMigration::Legacy, RelSizeMigration::Legacy) => {
|
||||
// tenant config didn't enable it and we didn't write any reldir_v2 key yet
|
||||
Ok(RelDirMode {
|
||||
current_status: RelSizeMigration::Legacy,
|
||||
initialize: false,
|
||||
disable_v1: false,
|
||||
})
|
||||
}
|
||||
(false, status @ RelSizeMigration::Migrating | status @ RelSizeMigration::Migrated) => {
|
||||
// index_part already persisted that the timeline has enabled rel_size_v2
|
||||
(
|
||||
RelSizeMigration::Legacy,
|
||||
current_status @ RelSizeMigration::Migrating
|
||||
| current_status @ RelSizeMigration::Migrated,
|
||||
) => {
|
||||
// already persisted that the timeline has enabled rel_size_v2, cannot rollback
|
||||
Ok(RelDirMode {
|
||||
current_status: status,
|
||||
current_status,
|
||||
initialize: false,
|
||||
disable_v1: false,
|
||||
})
|
||||
}
|
||||
(true, RelSizeMigration::Legacy) => {
|
||||
(
|
||||
expected_status @ RelSizeMigration::Migrating
|
||||
| expected_status @ RelSizeMigration::Migrated,
|
||||
RelSizeMigration::Legacy,
|
||||
) => {
|
||||
// The first time we enable it, we need to persist it in `index_part.json`
|
||||
// The caller should update the reldir status once the initialization is done.
|
||||
//
|
||||
// Only initialize the v2 keyspace on new relation creation. No initialization
|
||||
// during `timeline_create` (TODO: fix this, we should allow, but currently it
|
||||
// hits consistency issues).
|
||||
|
||||
Ok(RelDirMode {
|
||||
current_status: RelSizeMigration::Legacy,
|
||||
initialize: is_create && !self.is_importing_pgdata,
|
||||
initialize: can_update,
|
||||
disable_v1: can_update && expected_status == RelSizeMigration::Migrated,
|
||||
})
|
||||
}
|
||||
(true, status @ RelSizeMigration::Migrating | status @ RelSizeMigration::Migrated) => {
|
||||
// index_part already persisted that the timeline has enabled rel_size_v2
|
||||
// and we don't need to do anything
|
||||
(RelSizeMigration::Migrating, current_status @ RelSizeMigration::Migrating)
|
||||
| (RelSizeMigration::Migrated, current_status @ RelSizeMigration::Migrated)
|
||||
| (RelSizeMigration::Migrating, current_status @ RelSizeMigration::Migrated) => {
|
||||
// Keep the current state
|
||||
Ok(RelDirMode {
|
||||
current_status: status,
|
||||
current_status,
|
||||
initialize: false,
|
||||
disable_v1: false,
|
||||
})
|
||||
}
|
||||
(RelSizeMigration::Migrated, RelSizeMigration::Migrating) => {
|
||||
// Switch to v2-only mode
|
||||
Ok(RelDirMode {
|
||||
current_status: RelSizeMigration::Migrating,
|
||||
initialize: false,
|
||||
disable_v1: can_update,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2137,8 +2184,8 @@ impl DatadirModification<'_> {
|
||||
ctx: &RequestContext,
|
||||
) -> Result<(), WalIngestError> {
|
||||
let v2_mode = self
|
||||
.maybe_enable_rel_size_v2(false)
|
||||
.map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
|
||||
.maybe_enable_rel_size_v2(self.lsn, false)
|
||||
.map_err(WalIngestErrorKind::RelSizeV2Error)?;
|
||||
|
||||
// Add it to the directory (if it doesn't exist already)
|
||||
let buf = self.get(DBDIR_KEY, ctx).await?;
|
||||
@@ -2290,15 +2337,6 @@ impl DatadirModification<'_> {
|
||||
sparse_rel_dir_key,
|
||||
Value::Image(RelDirExists::Exists.encode()),
|
||||
);
|
||||
tracing::info!(
|
||||
"migrated rel_size_v2: {}",
|
||||
RelTag {
|
||||
spcnode,
|
||||
dbnode,
|
||||
relnode,
|
||||
forknum
|
||||
}
|
||||
);
|
||||
rel_cnt += 1;
|
||||
}
|
||||
}
|
||||
@@ -2307,12 +2345,25 @@ impl DatadirModification<'_> {
|
||||
self.lsn,
|
||||
rel_cnt
|
||||
);
|
||||
self.tline
|
||||
.update_rel_size_v2_status(RelSizeMigration::Migrating, Some(self.lsn))
|
||||
.map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
|
||||
Ok::<_, WalIngestError>(())
|
||||
}
|
||||
|
||||
async fn put_rel_creation_v1_placeholder(
|
||||
&mut self,
|
||||
rel: RelTag,
|
||||
dbdir_exists: bool,
|
||||
_ctx: &RequestContext,
|
||||
) -> Result<(), WalIngestError> {
|
||||
if !dbdir_exists {
|
||||
let rel_dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode);
|
||||
self.put(
|
||||
rel_dir_key,
|
||||
Value::Image(Bytes::from(RelDirectory::ser(&RelDirectory::default())?)),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_rel_creation_v1(
|
||||
&mut self,
|
||||
rel: RelTag,
|
||||
@@ -2391,26 +2442,26 @@ impl DatadirModification<'_> {
|
||||
}
|
||||
// It's possible that this is the first rel for this db in this
|
||||
// tablespace. Create the reldir entry for it if so.
|
||||
let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await?)?;
|
||||
let mut dbdir: DbDirectory = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await?)?;
|
||||
let mut is_dbdir_dirty = false;
|
||||
|
||||
let dbdir_exists =
|
||||
if let hash_map::Entry::Vacant(e) = dbdir.dbdirs.entry((rel.spcnode, rel.dbnode)) {
|
||||
// Didn't exist. Update dbdir
|
||||
e.insert(false);
|
||||
let buf = DbDirectory::ser(&dbdir)?;
|
||||
self.pending_directory_entries.push((
|
||||
DirectoryKind::Db,
|
||||
MetricsUpdate::Set(dbdir.dbdirs.len() as u64),
|
||||
));
|
||||
self.put(DBDIR_KEY, Value::Image(buf.into()));
|
||||
is_dbdir_dirty = true;
|
||||
false
|
||||
} else {
|
||||
true
|
||||
};
|
||||
|
||||
let mut v2_mode = self
|
||||
.maybe_enable_rel_size_v2(true)
|
||||
.map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
|
||||
.maybe_enable_rel_size_v2(self.lsn, true)
|
||||
.map_err(WalIngestErrorKind::RelSizeV2Error)?;
|
||||
|
||||
if v2_mode.initialize {
|
||||
if let Err(e) = self.initialize_rel_size_v2_keyspace(ctx, &dbdir).await {
|
||||
@@ -2418,11 +2469,30 @@ impl DatadirModification<'_> {
|
||||
// TODO: circuit breaker so that it won't retry forever
|
||||
} else {
|
||||
v2_mode.current_status = RelSizeMigration::Migrating;
|
||||
self.tline
|
||||
.update_rel_size_v2_status(RelSizeMigration::Migrating, Some(self.lsn))
|
||||
.map_err(WalIngestErrorKind::RelSizeV2Error)?;
|
||||
}
|
||||
}
|
||||
|
||||
if v2_mode.disable_v1 {
|
||||
v2_mode.current_status = RelSizeMigration::Migrated;
|
||||
self.tline
|
||||
.update_rel_size_v2_status(RelSizeMigration::Migrated, Some(self.lsn))
|
||||
.map_err(WalIngestErrorKind::RelSizeV2Error)?;
|
||||
}
|
||||
|
||||
if is_dbdir_dirty {
|
||||
let buf = DbDirectory::ser(&dbdir)?;
|
||||
self.put(DBDIR_KEY, Value::Image(buf.into()));
|
||||
}
|
||||
|
||||
if v2_mode.current_status != RelSizeMigration::Migrated {
|
||||
self.put_rel_creation_v1(rel, dbdir_exists, ctx).await?;
|
||||
} else {
|
||||
// collect_keyspace depends on the rel_dir_to_key to be present, so we need to create a placeholder
|
||||
self.put_rel_creation_v1_placeholder(rel, dbdir_exists, ctx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if v2_mode.current_status != RelSizeMigration::Legacy {
|
||||
@@ -2581,8 +2651,8 @@ impl DatadirModification<'_> {
|
||||
ctx: &RequestContext,
|
||||
) -> Result<(), WalIngestError> {
|
||||
let v2_mode = self
|
||||
.maybe_enable_rel_size_v2(false)
|
||||
.map_err(WalIngestErrorKind::MaybeRelSizeV2Error)?;
|
||||
.maybe_enable_rel_size_v2(self.lsn, false)
|
||||
.map_err(WalIngestErrorKind::RelSizeV2Error)?;
|
||||
match v2_mode.current_status {
|
||||
RelSizeMigration::Legacy => {
|
||||
self.put_rel_drop_v1(drop_relations, ctx).await?;
|
||||
@@ -2600,6 +2670,12 @@ impl DatadirModification<'_> {
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(WalIngestError {
|
||||
kind: WalIngestErrorKind::Cancelled,
|
||||
..
|
||||
}) => {
|
||||
// Cancellation errors are fine to ignore, do not log.
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("error dropping rels: {}", e);
|
||||
}
|
||||
|
||||
@@ -115,8 +115,6 @@ pub struct IndexPart {
|
||||
#[serde(skip_serializing_if = "Option::is_none", default)]
|
||||
pub(crate) marked_invisible_at: Option<NaiveDateTime>,
|
||||
|
||||
/// The LSN at which we started the rel size migration. Accesses below this LSN should be
|
||||
/// processed with the v1 read path. Usually this LSN should be set together with `rel_size_migration`.
|
||||
#[serde(skip_serializing_if = "Option::is_none", default)]
|
||||
pub(crate) rel_size_migrated_at: Option<Lsn>,
|
||||
}
|
||||
|
||||
@@ -2886,12 +2886,23 @@ impl Timeline {
|
||||
/// Returns `true` if the rel_size_v2 config is enabled. NOTE: the write path and read path
|
||||
/// should look at `get_rel_size_v2_status()` to get the actual status of the timeline. It is
|
||||
/// possible that the index part persists the state while the config doesn't get persisted.
|
||||
pub(crate) fn get_rel_size_v2_enabled(&self) -> bool {
|
||||
pub(crate) fn get_rel_size_v2_expected_state(&self) -> RelSizeMigration {
|
||||
let tenant_conf = self.tenant_conf.load();
|
||||
tenant_conf
|
||||
let v2_enabled = tenant_conf
|
||||
.tenant_conf
|
||||
.rel_size_v2_enabled
|
||||
.unwrap_or(self.conf.default_tenant_conf.rel_size_v2_enabled)
|
||||
.unwrap_or(self.conf.default_tenant_conf.rel_size_v2_enabled);
|
||||
let v1_access_disabled = tenant_conf
|
||||
.tenant_conf
|
||||
.rel_size_v1_access_disabled
|
||||
.unwrap_or(self.conf.default_tenant_conf.rel_size_v1_access_disabled);
|
||||
|
||||
match (v2_enabled, v1_access_disabled) {
|
||||
(true, false) => RelSizeMigration::Migrating,
|
||||
(true, true) => RelSizeMigration::Migrated,
|
||||
(false, true) => RelSizeMigration::Legacy, // This should never happen
|
||||
(false, false) => RelSizeMigration::Legacy,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn get_rel_size_v2_status(&self) -> (RelSizeMigration, Option<Lsn>) {
|
||||
|
||||
@@ -135,7 +135,7 @@ pub enum WalIngestErrorKind {
|
||||
#[error(transparent)]
|
||||
EncodeAuxFileError(anyhow::Error),
|
||||
#[error(transparent)]
|
||||
MaybeRelSizeV2Error(anyhow::Error),
|
||||
RelSizeV2Error(anyhow::Error),
|
||||
|
||||
#[error("timeline shutting down")]
|
||||
Cancelled,
|
||||
|
||||
@@ -1312,8 +1312,8 @@ class NeonEnv:
|
||||
)
|
||||
|
||||
tenant_config = ps_cfg.setdefault("tenant_config", {})
|
||||
# This feature is pending rollout.
|
||||
# tenant_config["rel_size_v2_enabled"] = True
|
||||
# Enable relsize_v2 by default in tests.
|
||||
tenant_config["rel_size_v2_enabled"] = True
|
||||
|
||||
# Test authors tend to forget about the default 10min initial lease deadline
|
||||
# when writing tests, which turns their immediate gc requests via mgmt API
|
||||
|
||||
@@ -80,7 +80,12 @@ def test_perf_simple_many_relations_reldir(
|
||||
"""
|
||||
Test creating many relations in a single database.
|
||||
"""
|
||||
env = neon_env_builder.init_start(initial_tenant_conf={"rel_size_v2_enabled": reldir != "v1"})
|
||||
env = neon_env_builder.init_start(
|
||||
initial_tenant_conf={
|
||||
"rel_size_v2_enabled": reldir != "v1",
|
||||
"rel_size_v1_access_disabled": reldir == "v2",
|
||||
}
|
||||
)
|
||||
ep = env.endpoints.create_start(
|
||||
"main",
|
||||
config_lines=[
|
||||
@@ -108,10 +113,6 @@ def test_perf_simple_many_relations_reldir(
|
||||
== "migrating"
|
||||
)
|
||||
elif reldir == "v2":
|
||||
# only read/write to the v2 keyspace
|
||||
env.pageserver.http_client().timeline_patch_index_part(
|
||||
env.initial_tenant, env.initial_timeline, {"rel_size_migration": "migrated"}
|
||||
)
|
||||
assert (
|
||||
env.pageserver.http_client().timeline_detail(env.initial_tenant, env.initial_timeline)[
|
||||
"rel_size_migration"
|
||||
|
||||
@@ -183,7 +183,7 @@ def test_fully_custom_config(positive_env: NeonEnv):
|
||||
"lsn_lease_length": "1m",
|
||||
"lsn_lease_length_for_ts": "5s",
|
||||
"timeline_offloading": False,
|
||||
"rel_size_v2_enabled": True,
|
||||
"rel_size_v2_enabled": False,
|
||||
"relsize_snapshot_cache_capacity": 10000,
|
||||
"gc_compaction_enabled": False,
|
||||
"gc_compaction_verification": False,
|
||||
|
||||
@@ -12,10 +12,12 @@ from fixtures.log_helper import log
|
||||
from fixtures.neon_fixtures import (
|
||||
NeonEnvBuilder,
|
||||
generate_uploads_and_deletions,
|
||||
wait_for_last_flush_lsn,
|
||||
)
|
||||
from fixtures.pageserver.http import PageserverApiException
|
||||
from fixtures.utils import skip_in_debug_build, wait_until
|
||||
from fixtures.workload import Workload
|
||||
from test_pg_regress import patch_tenant_conf
|
||||
|
||||
AGGRESSIVE_COMPACTION_TENANT_CONF = {
|
||||
# Disable gc and compaction. The test runs compaction manually.
|
||||
@@ -1101,7 +1103,8 @@ def test_image_consistent_lsn(neon_env_builder: NeonEnvBuilder):
|
||||
neon_env_builder.num_pageservers = 2
|
||||
neon_env_builder.num_safekeepers = 1
|
||||
env = neon_env_builder.init_start(
|
||||
initial_tenant_conf=tenant_conf,
|
||||
# We have to run this test with v1 to avoid v2 migration producing too many delta layers.
|
||||
initial_tenant_conf=patch_tenant_conf(tenant_conf, "v1"),
|
||||
initial_tenant_shard_count=4,
|
||||
initial_tenant_shard_stripe_size=1,
|
||||
)
|
||||
@@ -1116,6 +1119,8 @@ def test_image_consistent_lsn(neon_env_builder: NeonEnvBuilder):
|
||||
f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))", log_query=False
|
||||
)
|
||||
|
||||
wait_for_last_flush_lsn(env, endpoint, env.initial_tenant, env.initial_timeline)
|
||||
|
||||
response = env.storage_controller.tenant_timeline_describe(tenant_id, timeline_id)
|
||||
shards = response["shards"]
|
||||
for shard in shards:
|
||||
|
||||
@@ -581,6 +581,10 @@ def test_historic_storage_formats(
|
||||
# This dataset was created at a time where we decided to migrate to v2 reldir by simply disabling writes to v1
|
||||
# and starting writing to v2. This was too risky and we have reworked the migration plan. Therefore, we should
|
||||
# opt in full relv2 mode for this dataset.
|
||||
env.pageserver.http_client().patch_tenant_config(
|
||||
dataset.tenant_id,
|
||||
{"rel_size_v1_access_disabled": True, "rel_size_v2_enabled": True},
|
||||
)
|
||||
for timeline in timelines:
|
||||
env.pageserver.http_client().timeline_patch_index_part(
|
||||
dataset.tenant_id,
|
||||
|
||||
@@ -123,9 +123,15 @@ def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: End
|
||||
def patch_tenant_conf(tenant_conf: dict[str, Any], reldir_type: str) -> dict[str, Any]:
|
||||
tenant_conf = tenant_conf.copy()
|
||||
if reldir_type == "v2":
|
||||
tenant_conf["rel_size_v2_enabled"] = "true"
|
||||
tenant_conf["rel_size_v2_enabled"] = True
|
||||
tenant_conf["rel_size_v1_access_disabled"] = True
|
||||
elif reldir_type == "v1v2":
|
||||
tenant_conf["rel_size_v2_enabled"] = True
|
||||
tenant_conf["rel_size_v1_access_disabled"] = False
|
||||
elif reldir_type == "v1":
|
||||
tenant_conf["rel_size_v2_enabled"] = False
|
||||
else:
|
||||
tenant_conf["rel_size_v2_enabled"] = "false"
|
||||
raise ValueError(f"Invalid reldir_type: {reldir_type}")
|
||||
return tenant_conf
|
||||
|
||||
|
||||
@@ -133,7 +139,7 @@ def patch_tenant_conf(tenant_conf: dict[str, Any], reldir_type: str) -> dict[str
|
||||
#
|
||||
@pytest.mark.timeout(3000) # Contains many sub-tests, is slow in debug builds
|
||||
@pytest.mark.parametrize("shard_count", [None, 4])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v1v2", "v2"])
|
||||
def test_pg_regress(
|
||||
neon_env_builder: NeonEnvBuilder,
|
||||
test_output_dir: Path,
|
||||
@@ -213,7 +219,7 @@ def test_pg_regress(
|
||||
#
|
||||
@pytest.mark.timeout(1500) # Contains many sub-tests, is slow in debug builds
|
||||
@pytest.mark.parametrize("shard_count", [None, 4])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v1v2", "v2"])
|
||||
def test_isolation(
|
||||
neon_env_builder: NeonEnvBuilder,
|
||||
test_output_dir: Path,
|
||||
@@ -293,7 +299,7 @@ def test_isolation(
|
||||
# Run extra Neon-specific pg_regress-based tests. The tests and their
|
||||
# schedule file are in the sql_regress/ directory.
|
||||
@pytest.mark.parametrize("shard_count", [None, 4])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v1v2", "v2"])
|
||||
def test_sql_regress(
|
||||
neon_env_builder: NeonEnvBuilder,
|
||||
test_output_dir: Path,
|
||||
@@ -416,7 +422,7 @@ def test_max_wal_rate(neon_simple_env: NeonEnv):
|
||||
|
||||
|
||||
@skip_in_debug_build("only run with release build")
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
|
||||
@pytest.mark.parametrize("reldir_type", ["v1", "v1v2", "v2"])
|
||||
def test_tx_abort_with_many_relations(
|
||||
neon_env_builder: NeonEnvBuilder,
|
||||
reldir_type: str,
|
||||
|
||||
Reference in New Issue
Block a user