rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-31 20:10:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pageserver): persist timeline invisible flag (#11331)

Browse Source 
## Problem

part of https://github.com/neondatabase/neon/issues/11279

## Summary of changes

The invisible flag is used to exclude a timeline from synthetic size
calculation. For the first step, let's persist this flag. Most of the
code are following the `is_archived` modification flow.

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-03-20 14:39:08 -04:00
committed by GitHub
parent 53f54ba37a
commit bae9b9acdc
4 changed files with 192 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
libs/pageserver_api/src/models.rs
View File

@@ -1364,6 +1364,12 @@ pub enum TimelineArchivalState {
Unarchived,
}
#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
pub enum TimelineVisibilityState {
Visible,
Invisible,
}
#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
pub struct TimelineArchivalConfigRequest {
pub state: TimelineArchivalState,
@@ -1496,6 +1502,9 @@ pub struct TimelineInfo {
/// The status of the rel_size migration.
pub rel_size_migration: Option<RelSizeMigration>,
/// Whether the timeline is invisible in synthetic size calculations.
pub is_invisible: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]

32
pageserver/src/http/routes.rs
View File

@@ -37,8 +37,8 @@ use pageserver_api::models::{
TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateRequestMode,
TimelineCreateRequestModeImportPgdata, TimelineGcRequest, TimelineInfo,
TimelinePatchIndexPartRequest, TimelinesInfoAndOffloaded, TopTenantShardItem,
TopTenantShardsRequest, TopTenantShardsResponse,
TimelinePatchIndexPartRequest, TimelineVisibilityState, TimelinesInfoAndOffloaded,
TopTenantShardItem, TopTenantShardsRequest, TopTenantShardsResponse,
};
use pageserver_api::shard::{ShardCount, TenantShardId};
use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
@@ -439,6 +439,7 @@ async fn build_timeline_info_common(
let remote_consistent_lsn_visible = timeline
.get_remote_consistent_lsn_visible()
.unwrap_or(Lsn(0));
let is_invisible = timeline.remote_client.is_invisible().unwrap_or(false);
let walreceiver_status = timeline.walreceiver_status();
@@ -482,6 +483,7 @@ async fn build_timeline_info_common(
state,
is_archived: Some(is_archived),
rel_size_migration: Some(timeline.get_rel_size_v2_status()),
is_invisible: Some(is_invisible),
walreceiver_status,
};
@@ -2333,6 +2335,28 @@ async fn timeline_compact_handler(
.await
}
async fn timeline_mark_invisible_handler(
request: Request<Body>,
_cancel: CancellationToken,
) -> Result<Response<Body>, ApiError> {
let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
check_permission(&request, Some(tenant_shard_id.tenant_id))?;
let state = get_state(&request);
async {
let tenant = state
.tenant_manager
.get_attached_tenant_shard(tenant_shard_id)?;
let timeline = tenant.get_timeline(timeline_id, true)?;
timeline.remote_client.schedule_index_upload_for_timeline_invisible_state(TimelineVisibilityState::Invisible).map_err(ApiError::InternalServerError)?;
json_response(StatusCode::OK, ())
}
.instrument(info_span!("manual_timeline_mark_invisible", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id))
.await
}
// Run offload immediately on given timeline.
async fn timeline_offload_handler(
request: Request<Body>,
@@ -3750,6 +3774,10 @@ pub fn make_router(
"/v1/tenant/:tenant_shard_id/timeline/:timeline_id/offload",
|r| testing_api_handler("attempt timeline offload", r, timeline_offload_handler),
)
.put(
"/v1/tenant/:tenant_shard_id/timeline/:timeline_id/mark_invisible",
|r| testing_api_handler("mark timeline invisible", r, timeline_mark_invisible_handler),
)
.put(
"/v1/tenant/:tenant_shard_id/timeline/:timeline_id/checkpoint",
|r| testing_api_handler("run timeline checkpoint", r, timeline_checkpoint_handler),

43
pageserver/src/tenant/remote_timeline_client.rs
View File

@@ -194,7 +194,7 @@ pub(crate) use download::{
};
use index::GcCompactionState;
pub(crate) use index::LayerFileMetadata;
use pageserver_api::models::{RelSizeMigration, TimelineArchivalState};
use pageserver_api::models::{RelSizeMigration, TimelineArchivalState, TimelineVisibilityState};
use pageserver_api::shard::{ShardIndex, TenantShardId};
use regex::Regex;
use remote_storage::{
@@ -573,6 +573,16 @@ impl RemoteTimelineClient {
.ok()
}
/// Returns true if the timeline is invisible in synthetic size calculations.
pub(crate) fn is_invisible(&self) -> Option<bool> {
self.upload_queue
.lock()
.unwrap()
.initialized_mut()
.map(|q| q.clean.0.marked_invisible_at.is_some())
.ok()
}
/// Returns `Ok(Some(timestamp))` if the timeline has been archived, `Ok(None)` if the timeline hasn't been archived.
///
/// Return Err(_) if the remote index_part hasn't been downloaded yet, or the timeline hasn't been stopped yet.
@@ -845,6 +855,37 @@ impl RemoteTimelineClient {
Ok(need_wait)
}
pub(crate) fn schedule_index_upload_for_timeline_invisible_state(
self: &Arc<Self>,
state: TimelineVisibilityState,
) -> anyhow::Result<()> {
let mut guard = self.upload_queue.lock().unwrap();
let upload_queue = guard.initialized_mut()?;
fn need_change(
marked_invisible_at: &Option<NaiveDateTime>,
state: TimelineVisibilityState,
) -> Option<bool> {
match (marked_invisible_at, state) {
(Some(_), TimelineVisibilityState::Invisible) => Some(false),
(None, TimelineVisibilityState::Invisible) => Some(true),
(Some(_), TimelineVisibilityState::Visible) => Some(false),
(None, TimelineVisibilityState::Visible) => Some(true),
}
}
let need_upload_scheduled = need_change(&upload_queue.dirty.marked_invisible_at, state);
if let Some(marked_invisible_at_set) = need_upload_scheduled {
let intended_marked_invisible_at =
marked_invisible_at_set.then(|| Utc::now().naive_utc());
upload_queue.dirty.marked_invisible_at = intended_marked_invisible_at;
self.schedule_index_upload(upload_queue);
}
Ok(())
}
/// Shuts the timeline client down, but only if the timeline is archived.
///
/// This function and [`Self::schedule_index_upload_for_timeline_archival_state`] use the

115
pageserver/src/tenant/remote_timeline_client/index.rs
View File

@@ -110,6 +110,10 @@ pub struct IndexPart {
/// just the specific use case here; it needs a new name.
#[serde(skip_serializing_if = "Option::is_none", default)]
pub(crate) gc_compaction: Option<GcCompactionState>,
/// The timestamp when the timeline was marked invisible in synthetic size calculations.
#[serde(skip_serializing_if = "Option::is_none", default)]
pub(crate) marked_invisible_at: Option<NaiveDateTime>,
}
#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
@@ -137,10 +141,11 @@ impl IndexPart {
/// - 11: +rel_size_migration
/// - 12: +l2_lsn
/// - 13: +gc_compaction
const LATEST_VERSION: usize = 13;
/// - 14: +marked_invisible_at
const LATEST_VERSION: usize = 14;
// Versions we may see when reading from a bucket.
pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];
pub const FILE_NAME: &'static str = "index_part.json";
@@ -159,6 +164,7 @@ impl IndexPart {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
}
}
@@ -468,6 +474,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -516,6 +523,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -565,6 +573,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -617,6 +626,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let empty_layers_parsed = IndexPart::from_json_bytes(empty_layers_json.as_bytes()).unwrap();
@@ -664,6 +674,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -714,6 +725,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -769,6 +781,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -829,6 +842,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -890,6 +904,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -956,6 +971,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -1035,6 +1051,7 @@ mod tests {
rel_size_migration: None,
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -1115,6 +1132,7 @@ mod tests {
rel_size_migration: Some(RelSizeMigration::Legacy),
l2_lsn: None,
gc_compaction: None,
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
@@ -1124,7 +1142,7 @@ mod tests {
#[test]
fn v12_v13_l2_gc_ompaction_is_parsed() {
let example = r#"{
"version": 12,
"version": 13,
"layer_metadata":{
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
@@ -1160,7 +1178,7 @@ mod tests {
}"#;
let expected = IndexPart {
version: 12,
version: 13,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1201,6 +1219,95 @@ mod tests {
gc_compaction: Some(GcCompactionState {
last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
}),
marked_invisible_at: None,
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();
assert_eq!(part, expected);
}
#[test]
fn v14_marked_invisible_at_is_parsed() {
let example = r#"{
"version": 14,
"layer_metadata":{
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
"000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 }
},
"disk_consistent_lsn":"0/16960E8",
"metadata": {
"disk_consistent_lsn": "0/16960E8",
"prev_record_lsn": "0/1696070",
"ancestor_timeline": "e45a7f37d3ee2ff17dc14bf4f4e3f52e",
"ancestor_lsn": "0/0",
"latest_gc_cutoff_lsn": "0/1696070",
"initdb_lsn": "0/1696070",
"pg_version": 14
},
"gc_blocking": {
"started_at": "2024-07-19T09:00:00.123",
"reasons": ["DetachAncestor"]
},
"import_pgdata": {
"V1": {
"Done": {
"idempotency_key": "specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5",
"started_at": "2024-11-13T09:23:42.123",
"finished_at": "2024-11-13T09:42:23.123"
}
}
},
"rel_size_migration": "legacy",
"l2_lsn": "0/16960E8",
"gc_compaction": {
"last_completed_lsn": "0/16960E8"
},
"marked_invisible_at": "2023-07-31T09:00:00.123"
}"#;
let expected = IndexPart {
version: 14,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
generation: Generation::none(),
shard: ShardIndex::unsharded()
}),
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), LayerFileMetadata {
file_size: 9007199254741001,
generation: Generation::none(),
shard: ShardIndex::unsharded()
})
]),
disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
metadata: TimelineMetadata::new(
Lsn::from_str("0/16960E8").unwrap(),
Some(Lsn::from_str("0/1696070").unwrap()),
Some(TimelineId::from_str("e45a7f37d3ee2ff17dc14bf4f4e3f52e").unwrap()),
Lsn::INVALID,
Lsn::from_str("0/1696070").unwrap(),
Lsn::from_str("0/1696070").unwrap(),
14,
).with_recalculated_checksum().unwrap(),
deleted_at: None,
lineage: Default::default(),
gc_blocking: Some(GcBlocking {
started_at: parse_naive_datetime("2024-07-19T09:00:00.123000000"),
reasons: enumset::EnumSet::from_iter([GcBlockingReason::DetachAncestor]),
}),
last_aux_file_policy: Default::default(),
archived_at: None,
import_pgdata: Some(import_pgdata::index_part_format::Root::V1(import_pgdata::index_part_format::V1::Done(import_pgdata::index_part_format::Done{
started_at: parse_naive_datetime("2024-11-13T09:23:42.123000000"),
finished_at: parse_naive_datetime("2024-11-13T09:42:23.123000000"),
idempotency_key: import_pgdata::index_part_format::IdempotencyKey::new("specified-by-client-218a5213-5044-4562-a28d-d024c5f057f5".to_string()),
}))),
rel_size_migration: Some(RelSizeMigration::Legacy),
l2_lsn: Some("0/16960E8".parse::<Lsn>().unwrap()),
gc_compaction: Some(GcCompactionState {
last_completed_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
}),
marked_invisible_at: Some(parse_naive_datetime("2023-07-31T09:00:00.123000000")),
};
let part = IndexPart::from_json_bytes(example.as_bytes()).unwrap();