rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-10-19 10:13:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Port per-tenant upload queue and startup changes from #2595

Browse Source 
This is a part of https://github.com/neondatabase/neon/pull/2595.
It takes out switch to per tenant upload queue and changes to pageserver
startup sequence because these two are highly interleaved with each
other. I'm still not happy with the size of the diff, but splitting
it even more will probably consume even more time. Ideally we should
do it, but this patch isis already a step forward and should be
easier to get this patch in yet still quite difficult.
Mainly because of the size and fixes for existing concerns which
will extend the diff even further

Co-authored-by: Heikki Linnakangas <heikki@neon.tech>
This commit is contained in:
Dmitry Rodionov
2022-11-16 14:42:15 +02:00
co-authored by Heikki Linnakangas
parent d013a2b227
commit 795c3ca131
18 changed files with 2317 additions and 4609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/pageserver_api/src/models.rs
+9 -6
View File
@@ -15,8 +15,14 @@ use bytes::{BufMut, Bytes, BytesMut};
/// A state of a tenant in pageserver's memory.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum TenantState {
// This tenant is being loaded from local disk
Loading,
// This tenant is being downloaded from cloud storage.
Attaching,
/// Tenant is fully operational, its background jobs might be running or not.
Active { background_jobs_running: bool },
Active {
background_jobs_running: bool,
},
/// A tenant is recognized by pageserver, but not yet ready to operate:
/// e.g. not present locally and being downloaded or being read into memory from the file system.
Paused,
@@ -144,7 +150,6 @@ pub struct TenantInfo {
pub id: TenantId,
pub state: TenantState,
pub current_physical_size: Option<u64>, // physical size is only included in `tenant_status` endpoint
pub has_in_progress_downloads: Option<bool>,
}
/// This represents the output of the "timeline_detail" and "timeline_list" API calls.
@@ -168,6 +173,8 @@ pub struct TimelineInfo {
pub latest_gc_cutoff_lsn: Lsn,
#[serde_as(as = "DisplayFromStr")]
pub disk_consistent_lsn: Lsn,
#[serde_as(as = "DisplayFromStr")]
pub remote_consistent_lsn: Lsn,
pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
pub current_logical_size_non_incremental: Option<u64>,
@@ -180,10 +187,6 @@ pub struct TimelineInfo {
pub last_received_msg_ts: Option<u128>,
pub pg_version: u32,
#[serde_as(as = "Option<DisplayFromStr>")]
pub remote_consistent_lsn: Option<Lsn>,
pub awaits_download: bool,
pub state: TimelineState,
// Some of the above fields are duplicated in 'local' and 'remote', for backwards-
pageserver/src/bin/pageserver.rs
+2 -2
View File
@@ -264,7 +264,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
})
.transpose()
.context("Failed to init generic remote storage")?;
let remote_index = {
{
let _rt_guard = BACKGROUND_RUNTIME.enter();
tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())?
};
@@ -276,7 +276,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
{
let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
let router = http::make_router(conf, auth.clone(), remote_index, remote_storage)?;
let router = http::make_router(conf, auth.clone(), remote_storage)?;
let service =
utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap();
let server = hyper::Server::from_tcp(http_listener)?
pageserver/src/http/routes.rs
+92 -238
View File
@@ -7,15 +7,12 @@ use remote_storage::GenericRemoteStorage;
use tokio::task::JoinError;
use tracing::*;
use super::models::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
use super::models::{
StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
TimelineCreateRequest,
LocalTimelineInfo, RemoteTimelineInfo, StatusResponse, TenantConfigRequest,
TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo,
};
use crate::pgdatadir_mapping::LsnForTimestamp;
use crate::storage_sync;
use crate::storage_sync::index::{RemoteIndex, RemoteTimeline};
use crate::tenant::{TenantState, Timeline};
use crate::tenant::Timeline;
use crate::tenant_config::TenantConfOpt;
use crate::{config::PageServerConf, tenant_mgr};
use utils::{
@@ -27,7 +24,7 @@ use utils::{
request::parse_request_param,
RequestExt, RouterBuilder,
},
id::{TenantId, TenantTimelineId, TimelineId},
id::{TenantId, TimelineId},
lsn::Lsn,
};
@@ -40,7 +37,6 @@ use crate::CheckpointConfig;
struct State {
conf: &'static PageServerConf,
auth: Option<Arc<JwtAuth>>,
remote_index: RemoteIndex,
allowlist_routes: Vec<Uri>,
remote_storage: Option<GenericRemoteStorage>,
}
@@ -49,7 +45,6 @@ impl State {
fn new(
conf: &'static PageServerConf,
auth: Option<Arc<JwtAuth>>,
remote_index: RemoteIndex,
remote_storage: Option<GenericRemoteStorage>,
) -> anyhow::Result<Self> {
let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
@@ -60,7 +55,6 @@ impl State {
conf,
auth,
allowlist_routes,
remote_index,
remote_storage,
})
}
@@ -80,12 +74,24 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
}
// Helper function to construct a TimelineInfo struct for a timeline
async fn build_timeline_info(
state: &State,
fn build_timeline_info(
timeline: &Arc<Timeline>,
include_non_incremental_logical_size: bool,
include_non_incremental_physical_size: bool,
) -> anyhow::Result<TimelineInfo> {
let mut info = build_timeline_info_common(timeline)?;
if include_non_incremental_logical_size {
info.current_logical_size_non_incremental =
Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?);
}
if include_non_incremental_physical_size {
info.current_physical_size_non_incremental =
Some(timeline.get_physical_size_non_incremental()?)
}
Ok(info)
}
fn build_timeline_info_common(timeline: &Arc<Timeline>) -> anyhow::Result<TimelineInfo> {
let last_record_lsn = timeline.get_last_record_lsn();
let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
let guard = timeline.last_received_wal.lock().unwrap();
@@ -100,22 +106,6 @@ async fn build_timeline_info(
}
};
let (remote_consistent_lsn, awaits_download) = if let Some(remote_entry) = state
.remote_index
.read()
.await
.timeline_entry(&TenantTimelineId {
tenant_id: timeline.tenant_id,
timeline_id: timeline.timeline_id,
}) {
(
Some(remote_entry.metadata.disk_consistent_lsn()),
remote_entry.awaits_download,
)
} else {
(None, false)
};
let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
let ancestor_lsn = match timeline.get_ancestor_lsn() {
Lsn(0) => None,
@@ -130,6 +120,7 @@ async fn build_timeline_info(
};
let current_physical_size = Some(timeline.get_physical_size());
let state = timeline.current_state();
let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
let info = TimelineInfo {
tenant_id: timeline.tenant_id,
@@ -137,28 +128,19 @@ async fn build_timeline_info(
ancestor_timeline_id,
ancestor_lsn,
disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
remote_consistent_lsn,
last_record_lsn,
prev_record_lsn: Some(timeline.get_prev_record_lsn()),
latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(),
current_logical_size,
current_physical_size,
current_logical_size_non_incremental: if include_non_incremental_logical_size {
Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?)
} else {
None
},
current_physical_size_non_incremental: if include_non_incremental_physical_size {
Some(timeline.get_physical_size_non_incremental()?)
} else {
None
},
current_logical_size_non_incremental: None,
current_physical_size_non_incremental: None,
wal_source_connstr,
last_received_msg_lsn,
last_received_msg_ts,
pg_version: timeline.pg_version,
remote_consistent_lsn,
awaits_download,
state,
// Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility
@@ -170,7 +152,7 @@ async fn build_timeline_info(
current_physical_size,
},
remote: RemoteTimelineInfo {
remote_consistent_lsn,
remote_consistent_lsn: Some(remote_consistent_lsn),
},
};
Ok(info)
@@ -187,34 +169,28 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
let request_data: TimelineCreateRequest = json_request(&mut request).await?;
check_permission(&request, Some(tenant_id))?;
let state = get_state(&request);
let new_timeline_id = request_data
.new_timeline_id
.unwrap_or_else(TimelineId::generate);
let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
let new_timeline_info = async {
match tenant.create_timeline(
request_data.new_timeline_id.map(TimelineId::from),
request_data.ancestor_timeline_id.map(TimelineId::from),
request_data.ancestor_start_lsn,
request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
).await {
Ok(Some(new_timeline)) => {
// Created. Construct a TimelineInfo for it.
let timeline_info = build_timeline_info(state, &new_timeline, false, false)
.await
.map_err(ApiError::InternalServerError)?;
Ok(Some(timeline_info))
}
Ok(None) => Ok(None), // timeline already exists
Err(err) => Err(ApiError::InternalServerError(err)),
match tenant.create_timeline(
new_timeline_id,
request_data.ancestor_timeline_id.map(TimelineId::from),
request_data.ancestor_start_lsn,
request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
)
.instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
.await {
Ok(Some(new_timeline)) => {
// Created. Construct a TimelineInfo for it.
let timeline_info = build_timeline_info_common(&new_timeline)
.map_err(ApiError::InternalServerError)?;
json_response(StatusCode::CREATED, timeline_info)
}
Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists
Err(err) => Err(ApiError::InternalServerError(err)),
}
.instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
.await?;
Ok(match new_timeline_info {
Some(info) => json_response(StatusCode::CREATED, info)?,
None => json_response(StatusCode::CONFLICT, ())?,
})
}
async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -225,22 +201,20 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
query_param_present(&request, "include-non-incremental-physical-size");
check_permission(&request, Some(tenant_id))?;
let state = get_state(&request);
let _entered = info_span!("timeline_list", tenant = %tenant_id).entered();
let timelines = info_span!("timeline_list", tenant = %tenant_id).in_scope(|| {
let timelines = {
let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
Ok(tenant.list_timelines())
})?;
tenant.list_timelines()
};
let mut response_data = Vec::with_capacity(timelines.len());
for timeline in timelines {
let timeline_info = build_timeline_info(
state,
&timeline,
include_non_incremental_logical_size,
include_non_incremental_physical_size,
)
.await
.context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
.map_err(ApiError::InternalServerError)?;
@@ -289,8 +263,6 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
query_param_present(&request, "include-non-incremental-physical-size");
check_permission(&request, Some(tenant_id))?;
let state = get_state(&request);
let timeline_info = async {
let timeline = tokio::task::spawn_blocking(move || {
tenant_mgr::get_tenant(tenant_id, true)?.get_timeline(timeline_id, false)
@@ -301,12 +273,10 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
let timeline = timeline.map_err(ApiError::NotFound)?;
let timeline_info = build_timeline_info(
state,
&timeline,
include_non_incremental_logical_size,
include_non_incremental_physical_size,
)
.await
.context("Failed to get local timeline info: {e:#}")
.map_err(ApiError::InternalServerError)?;
@@ -351,117 +321,28 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,
info!("Handling tenant attach {tenant_id}");
tokio::task::spawn_blocking(move || match tenant_mgr::get_tenant(tenant_id, false) {
Ok(tenant) => {
if tenant.list_timelines().is_empty() {
info!("Attaching to tenant {tenant_id} with zero timelines");
Ok(())
} else {
Err(ApiError::Conflict(
"Tenant is already present locally".to_owned(),
))
}
}
Err(_) => Ok(()),
})
.await
.map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
let state = get_state(&request);
let remote_index = &state.remote_index;
let mut index_accessor = remote_index.write().await;
if let Some(tenant_entry) = index_accessor.tenant_entry_mut(&tenant_id) {
if tenant_entry.has_in_progress_downloads() {
return Err(ApiError::Conflict(
"Tenant download is already in progress".to_string(),
));
}
for (timeline_id, remote_timeline) in tenant_entry.iter_mut() {
storage_sync::schedule_layer_download(tenant_id, *timeline_id);
remote_timeline.awaits_download = true;
}
return json_response(StatusCode::ACCEPTED, ());
}
// no tenant in the index, release the lock to make the potentially lengthy download operation
drop(index_accessor);
// download index parts for every tenant timeline
let remote_timelines = match gather_tenant_timelines_index_parts(state, tenant_id).await {
Ok(Some(remote_timelines)) => remote_timelines,
Ok(None) => return Err(ApiError::NotFound(anyhow!("Unknown remote tenant"))),
Err(e) => {
error!("Failed to retrieve remote tenant data: {:?}", e);
return Err(ApiError::NotFound(anyhow!(
"Failed to retrieve remote tenant"
)));
}
};
// recheck that download is not in progress because
// we've released the lock to avoid holding it during the download
let mut index_accessor = remote_index.write().await;
let tenant_entry = match index_accessor.tenant_entry_mut(&tenant_id) {
Some(tenant_entry) => {
if tenant_entry.has_in_progress_downloads() {
return Err(ApiError::Conflict(
"Tenant download is already in progress".to_string(),
));
}
tenant_entry
}
None => index_accessor.add_tenant_entry(tenant_id),
};
// populate remote index with the data from index part and create directories on the local filesystem
for (timeline_id, mut remote_timeline) in remote_timelines {
tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
if let Some(remote_storage) = &state.remote_storage {
// FIXME: distinguish between "Tenant already exists" and other errors
tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage)
.instrument(info_span!("tenant_attach", tenant = %tenant_id))
.await
.context("Failed to create new timeline directory")
.map_err(ApiError::InternalServerError)?;
remote_timeline.awaits_download = true;
tenant_entry.insert(timeline_id, remote_timeline);
// schedule actual download
storage_sync::schedule_layer_download(tenant_id, timeline_id);
} else {
return Err(ApiError::BadRequest(anyhow!(
"attach_tenant is possible because pageserver was configured without remote storage"
)));
}
json_response(StatusCode::ACCEPTED, ())
}
/// Note: is expensive from s3 access perspective,
/// for details see comment to `storage_sync::gather_tenant_timelines_index_parts`
async fn gather_tenant_timelines_index_parts(
state: &State,
tenant_id: TenantId,
) -> anyhow::Result<Option<Vec<(TimelineId, RemoteTimeline)>>> {
let index_parts = match state.remote_storage.as_ref() {
Some(storage) => {
storage_sync::gather_tenant_timelines_index_parts(state.conf, storage, tenant_id).await
}
None => return Ok(None),
}
.with_context(|| format!("Failed to download index parts for tenant {tenant_id}"))?;
let mut remote_timelines = Vec::with_capacity(index_parts.len());
for (timeline_id, index_part) in index_parts {
let timeline_path = state.conf.timeline_path(&timeline_id, &tenant_id);
let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
.with_context(|| {
format!("Failed to convert index part into remote timeline for timeline {tenant_id}/{timeline_id}")
})?;
remote_timelines.push((timeline_id, remote_timeline));
}
Ok(Some(remote_timelines))
}
async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
check_permission(&request, Some(tenant_id))?;
let state = get_state(&request);
tenant_mgr::delete_timeline(tenant_id, timeline_id)
.instrument(info_span!("timeline_delete", tenant = %tenant_id, timeline = %timeline_id))
.await
@@ -470,12 +351,6 @@ async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body
// it.
.map_err(ApiError::InternalServerError)?;
let mut remote_index = state.remote_index.write().await;
remote_index.remove_timeline_entry(TenantTimelineId {
tenant_id,
timeline_id,
});
json_response(StatusCode::OK, ())
}
@@ -492,22 +367,22 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
// Replace this with better handling once the error type permits it.
.map_err(ApiError::InternalServerError)?;
let mut remote_index = state.remote_index.write().await;
remote_index.remove_tenant_entry(&tenant_id);
json_response(StatusCode::OK, ())
}
async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
check_permission(&request, None)?;
let state = get_state(&request);
// clone to avoid holding the lock while awaiting for blocking task
let remote_index = state.remote_index.read().await.clone();
let response_data = tokio::task::spawn_blocking(move || {
let _enter = info_span!("tenant_list").entered();
crate::tenant_mgr::list_tenant_info(&remote_index)
tenant_mgr::list_tenants()
.iter()
.map(|(id, state)| TenantInfo {
id: *id,
state: *state,
current_physical_size: None,
})
.collect::<Vec<TenantInfo>>()
})
.await
.map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
@@ -519,58 +394,36 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
check_permission(&request, Some(tenant_id))?;
// if tenant is in progress of downloading it can be absent in global tenant map
let tenant = tenant_mgr::get_tenant(tenant_id, false);
let tenant_info = tokio::task::spawn_blocking(move || {
let _enter = info_span!("tenant_status_handler", tenant = %tenant_id).entered();
let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
let state = get_state(&request);
let remote_index = &state.remote_index;
let index_accessor = remote_index.read().await;
let has_in_progress_downloads = index_accessor
.tenant_entry(&tenant_id)
.map(|t| t.has_in_progress_downloads())
.unwrap_or_else(|| {
info!("Tenant {tenant_id} not found in remote index");
false
});
let (tenant_state, current_physical_size) = match tenant {
Ok(tenant) => {
let timelines = tenant.list_timelines();
// Calculate total physical size of all timelines
let mut current_physical_size = 0;
for timeline in timelines {
current_physical_size += timeline.get_physical_size();
}
(tenant.current_state(), Some(current_physical_size))
// Calculate total physical size of all timelines
let mut current_physical_size = 0;
for timeline in tenant.list_timelines().iter() {
current_physical_size += timeline.get_physical_size();
}
Err(e) => {
error!("Failed to get local tenant state: {e:#}");
if has_in_progress_downloads {
(TenantState::Paused, None)
} else {
(TenantState::Broken, None)
}
}
};
json_response(
StatusCode::OK,
TenantInfo {
let tenant_info = TenantInfo {
id: tenant_id,
state: tenant_state,
current_physical_size,
has_in_progress_downloads: Some(has_in_progress_downloads),
},
)
state: tenant.current_state(),
current_physical_size: Some(current_physical_size),
};
Ok::<_, anyhow::Error>(tenant_info)
})
.await
.map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?
.map_err(ApiError::InternalServerError)?;
json_response(StatusCode::OK, tenant_info)
}
async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
check_permission(&request, Some(tenant_id))?;
let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::InternalServerError)?;
let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::InternalServerError)?;
// this can be long operation, it currently is not backed by any request coalescing or similar
let inputs = tenant
@@ -618,8 +471,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
check_permission(&request, None)?;
let request_data: TenantCreateRequest = json_request(&mut request).await?;
println!("tenant create: {:?}", request_data.trace_read_requests);
let remote_index = get_state(&request).remote_index.clone();
let mut tenant_conf = TenantConfOpt::default();
if let Some(gc_period) = request_data.gc_period {
@@ -691,12 +542,17 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
let new_tenant_id = tokio::task::spawn_blocking(move || {
let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
let conf = get_config(&request);
let state = get_state(&request);
tenant_mgr::create_tenant(conf, tenant_conf, target_tenant_id, remote_index)
// FIXME: `create_tenant` can fail from both user and internal errors. Replace this
// with better error handling once the type permits it
.map_err(ApiError::InternalServerError)
tenant_mgr::create_tenant(
state.conf,
tenant_conf,
target_tenant_id,
state.remote_storage.as_ref(),
)
// FIXME: `create_tenant` can fail from both user and internal errors. Replace this
// with better error handling once the type permits it
.map_err(ApiError::InternalServerError)
})
.await
.map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
@@ -891,7 +747,6 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
pub fn make_router(
conf: &'static PageServerConf,
auth: Option<Arc<JwtAuth>>,
remote_index: RemoteIndex,
remote_storage: Option<GenericRemoteStorage>,
) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
let spec = include_bytes!("openapi_spec.yml");
@@ -928,8 +783,7 @@ pub fn make_router(
Ok(router
.data(Arc::new(
State::new(conf, auth, remote_index, remote_storage)
.context("Failed to initialize router state")?,
State::new(conf, auth, remote_storage).context("Failed to initialize router state")?,
))
.get("/v1/status", status_handler)
.put(
pageserver/src/lib.rs
+8 -18
View File
@@ -22,10 +22,7 @@ pub mod walreceiver;
pub mod walrecord;
pub mod walredo;
use std::collections::HashMap;
use tracing::info;
use utils::id::{TenantId, TimelineId};
use crate::task_mgr::TaskKind;
@@ -71,7 +68,7 @@ pub async fn shutdown_pageserver(exit_code: i32) {
//
// FIXME: Does this wait for the sync tasks to finish syncing what's queued up?
// Should it?
task_mgr::shutdown_tasks(Some(TaskKind::StorageSync), None, None).await;
task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await;
// Shut down the HTTP endpoint last, so that you can still check the server's
// status while it's shutting down.
@@ -106,24 +103,17 @@ fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds
}
}
/// A newtype to store arbitrary data grouped by tenant and timeline ids.
/// One could use [`utils::id::TenantTimelineId`] for grouping, but that would
/// not include the cases where a certain tenant has zero timelines.
/// This is sometimes important: a tenant could be registered during initial load from FS,
/// even if he has no timelines on disk.
#[derive(Debug)]
pub struct TenantTimelineValues<T>(HashMap<TenantId, HashMap<TimelineId, T>>);
impl<T> TenantTimelineValues<T> {
fn new() -> Self {
Self(HashMap::new())
}
}
/// A suffix to be used during file sync from the remote storage,
/// to ensure that we do not leave corrupted files that pretend to be layers.
const TEMP_FILE_SUFFIX: &str = "___temp";
pub fn is_temporary(path: &std::path::Path) -> bool {
match path.file_name() {
Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
None => false,
}
}
#[cfg(test)]
mod backoff_defaults_tests {
use super::*;
pageserver/src/storage_sync.rs
+806 -1756
View File
File diff suppressed because it is too large Load Diff
pageserver/src/storage_sync/delete.rs
+26 -224
View File
@@ -1,235 +1,37 @@
//! Timeline synchronization logic to delete a bulk of timeline's remote files from the remote storage.
use std::path::Path;
//! Helper functions to delete files from remote storage with a RemoteStorage
use anyhow::Context;
use futures::stream::{FuturesUnordered, StreamExt};
use tracing::{debug, error, info};
use std::path::Path;
use tracing::debug;
use crate::storage_sync::{SyncQueue, SyncTask};
use remote_storage::GenericRemoteStorage;
use utils::id::TenantTimelineId;
use super::{LayersDeletion, SyncData};
/// Attempts to remove the timleline layers from the remote storage.
/// If the task had not adjusted the metadata before, the deletion will fail.
pub(super) async fn delete_timeline_layers(
storage: &GenericRemoteStorage,
sync_queue: &SyncQueue,
sync_id: TenantTimelineId,
mut delete_data: SyncData<LayersDeletion>,
) -> bool {
if !delete_data.data.deletion_registered {
error!("Cannot delete timeline layers before the deletion metadata is not registered, reenqueueing");
delete_data.retries += 1;
sync_queue.push(sync_id, SyncTask::Delete(delete_data));
return false;
}
if delete_data.data.layers_to_delete.is_empty() {
info!("No layers to delete, skipping");
return true;
}
let layers_to_delete = delete_data
.data
.layers_to_delete
.drain()
.collect::<Vec<_>>();
debug!("Layers to delete: {layers_to_delete:?}");
info!("Deleting {} timeline layers", layers_to_delete.len());
let mut delete_tasks = layers_to_delete
.into_iter()
.map(|local_layer_path| async {
match remove_storage_object(storage, &local_layer_path).await {
Ok(()) => Ok(local_layer_path),
Err(e) => Err((e, local_layer_path)),
}
})
.collect::<FuturesUnordered<_>>();
let mut errored = false;
while let Some(deletion_result) = delete_tasks.next().await {
match deletion_result {
Ok(local_layer_path) => {
debug!(
"Successfully deleted layer {} for timeline {sync_id}",
local_layer_path.display()
);
delete_data.data.deleted_layers.insert(local_layer_path);
}
Err((e, local_layer_path)) => {
errored = true;
error!(
"Failed to delete layer {} for timeline {sync_id}: {e:?}",
local_layer_path.display()
);
delete_data.data.layers_to_delete.insert(local_layer_path);
}
}
}
if errored {
debug!("Reenqueuing failed delete task for timeline {sync_id}");
delete_data.retries += 1;
sync_queue.push(sync_id, SyncTask::Delete(delete_data));
} else {
info!("Successfully deleted all layers");
}
errored
}
async fn remove_storage_object(
pub(super) async fn delete_layer(
storage: &GenericRemoteStorage,
local_layer_path: &Path,
) -> anyhow::Result<()> {
let storage_path = storage
.remote_object_id(local_layer_path)
.with_context(|| {
async {
debug!(
"Deleting layer from remote storage: {:?}",
local_layer_path.display()
);
let storage_path = storage
.remote_object_id(local_layer_path)
.with_context(|| {
format!(
"Failed to get the layer storage path for local path '{}'",
local_layer_path.display()
)
})?;
// FIXME: If the deletion fails because the object already didn't exist,
// it would be good to just issue a warning but consider it success.
storage.delete(&storage_path).await.with_context(|| {
format!(
"Failed to get the layer storage path for local path '{}'",
local_layer_path.display()
"Failed to delete remote layer from storage at '{:?}'",
storage_path
)
})?;
storage.delete(&storage_path).await.with_context(|| {
format!(
"Failed to delete remote layer from storage at '{:?}'",
storage_path
)
})
}
#[cfg(test)]
mod tests {
use std::{collections::HashSet, num::NonZeroUsize};
use itertools::Itertools;
use tempfile::tempdir;
use tokio::fs;
use utils::lsn::Lsn;
use crate::{
storage_sync::test_utils::{create_local_timeline, dummy_metadata},
tenant::harness::{TenantHarness, TIMELINE_ID},
};
use remote_storage::{LocalFs, RemoteStorage};
use super::*;
#[tokio::test]
async fn delete_timeline_negative() -> anyhow::Result<()> {
let harness = TenantHarness::create("delete_timeline_negative")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_path_buf(),
harness.conf.workdir.clone(),
)?);
let deleted = delete_timeline_layers(
&storage,
&sync_queue,
sync_id,
SyncData {
retries: 1,
data: LayersDeletion {
deleted_layers: HashSet::new(),
layers_to_delete: HashSet::new(),
deletion_registered: false,
},
},
)
.await;
assert!(
!deleted,
"Should not start the deletion for task with delete metadata unregistered"
);
Ok(())
}
#[tokio::test]
async fn delete_timeline() -> anyhow::Result<()> {
let harness = TenantHarness::create("delete_timeline")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let layer_files = ["a", "b", "c", "d"];
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_path_buf(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let current_retries = 3;
let metadata = dummy_metadata(Lsn(0x30));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let timeline_upload =
create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
for (local_path, _metadata) in timeline_upload.layers_to_upload {
let remote_path =
local_storage.resolve_in_storage(&local_storage.remote_object_id(&local_path)?)?;
let remote_parent_dir = remote_path.parent().unwrap();
if !remote_parent_dir.exists() {
fs::create_dir_all(&remote_parent_dir).await?;
}
fs::copy(&local_path, &remote_path).await?;
}
assert_eq!(
local_storage
.list()
.await?
.into_iter()
.map(|remote_path| local_storage.local_path(&remote_path).unwrap())
.filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
.sorted()
.collect::<Vec<_>>(),
layer_files
.iter()
.map(|layer_str| layer_str.to_string())
.sorted()
.collect::<Vec<_>>(),
"Expect to have all layer files remotely before deletion"
);
let deleted = delete_timeline_layers(
&storage,
&sync_queue,
sync_id,
SyncData {
retries: current_retries,
data: LayersDeletion {
deleted_layers: HashSet::new(),
layers_to_delete: HashSet::from([
local_timeline_path.join("a"),
local_timeline_path.join("c"),
local_timeline_path.join("something_different"),
]),
deletion_registered: true,
},
},
)
.await;
assert!(deleted, "Should be able to delete timeline files");
assert_eq!(
local_storage
.list()
.await?
.into_iter()
.map(|remote_path| local_storage.local_path(&remote_path).unwrap())
.filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
.sorted()
.collect::<Vec<_>>(),
vec!["b".to_string(), "d".to_string()],
"Expect to have only non-deleted files remotely"
);
Ok(())
})
}
.await
}
pageserver/src/storage_sync/download.rs
+203 -641
View File
@@ -1,430 +1,154 @@
//! Timeline synchronization logic to fetch the layer files from remote storage into pageserver's local directory.
//! Helper functions to download files from remote storage with a RemoteStorage
use std::collections::HashSet;
use std::{
collections::{HashMap, HashSet},
fmt::Debug,
mem,
path::Path,
};
use anyhow::Context;
use anyhow::{bail, Context};
use futures::stream::{FuturesUnordered, StreamExt};
use tokio::fs;
use tokio::io::AsyncWriteExt;
use tracing::debug;
use crate::config::PageServerConf;
use crate::storage_sync::index::LayerFileMetadata;
use remote_storage::{DownloadError, GenericRemoteStorage};
use tokio::{
fs,
io::{self, AsyncWriteExt},
};
use tracing::{debug, error, info, warn};
use utils::crashsafe::path_with_suffix_extension;
use utils::id::{TenantId, TimelineId};
use crate::{
config::PageServerConf,
storage_sync::{index::LayerFileMetadata, SyncTask},
TEMP_FILE_SUFFIX,
};
use utils::{
crashsafe::path_with_suffix_extension,
id::{TenantId, TenantTimelineId, TimelineId},
};
use super::index::IndexPart;
use super::RelativePath;
use super::{
index::{IndexPart, RemoteTimeline},
LayersDownload, SyncData, SyncQueue,
};
// We collect timelines remotely available for each tenant
// in case we failed to gather all index parts (due to an error)
// Poisoned variant is returned.
// When data is received succesfully without errors Present variant is used.
pub enum TenantIndexParts {
Poisoned {
present: HashMap<TimelineId, IndexPart>,
missing: HashSet<TimelineId>,
},
Present(HashMap<TimelineId, IndexPart>),
async fn fsync_path(path: impl AsRef<std::path::Path>) -> Result<(), std::io::Error> {
fs::File::open(path).await?.sync_all().await
}
impl TenantIndexParts {
fn add_poisoned(&mut self, timeline_id: TimelineId) {
match self {
TenantIndexParts::Poisoned { missing, .. } => {
missing.insert(timeline_id);
}
TenantIndexParts::Present(present) => {
*self = TenantIndexParts::Poisoned {
present: mem::take(present),
missing: HashSet::from([timeline_id]),
}
}
}
}
}
impl Default for TenantIndexParts {
fn default() -> Self {
TenantIndexParts::Present(HashMap::default())
}
}
pub async fn download_index_parts(
conf: &'static PageServerConf,
storage: &GenericRemoteStorage,
keys: HashSet<TenantTimelineId>,
) -> HashMap<TenantId, TenantIndexParts> {
let mut index_parts: HashMap<TenantId, TenantIndexParts> = HashMap::new();
let mut part_downloads = keys
.into_iter()
.map(|id| async move { (id, download_index_part(conf, storage, id).await) })
.collect::<FuturesUnordered<_>>();
while let Some((id, part_upload_result)) = part_downloads.next().await {
match part_upload_result {
Ok(index_part) => {
debug!("Successfully fetched index part for {id}");
match index_parts.entry(id.tenant_id).or_default() {
TenantIndexParts::Poisoned { present, .. } => {
present.insert(id.timeline_id, index_part);
}
TenantIndexParts::Present(parts) => {
parts.insert(id.timeline_id, index_part);
}
}
}
Err(download_error) => {
match download_error {
DownloadError::NotFound => {
// thats ok because it means that we didnt upload something we have locally for example
}
e => {
let tenant_parts = index_parts.entry(id.tenant_id).or_default();
tenant_parts.add_poisoned(id.timeline_id);
error!(
"Failed to fetch index part for {id}: {e} poisoning tenant index parts"
);
}
}
}
}
}
index_parts
}
/// Note: The function is rather expensive from s3 access point of view, it will execute ceil(N/1000) + N requests.
/// At least one request to obtain a list of tenant timelines (more requests is there are more than 1000 timelines).
/// And then will attempt to download all index files that belong to these timelines.
pub async fn gather_tenant_timelines_index_parts(
conf: &'static PageServerConf,
storage: &GenericRemoteStorage,
tenant_id: TenantId,
) -> anyhow::Result<HashMap<TimelineId, IndexPart>> {
let tenant_path = conf.timelines_path(&tenant_id);
let timeline_sync_ids = get_timeline_sync_ids(storage, &tenant_path, tenant_id)
.await
.with_context(|| format!("Failed to list timeline sync ids for tenat {tenant_id}"))?;
match download_index_parts(conf, storage, timeline_sync_ids)
.await
.remove(&tenant_id)
.ok_or_else(|| anyhow::anyhow!("Missing tenant index parts. This is a bug."))?
{
TenantIndexParts::Poisoned { missing, .. } => {
anyhow::bail!("Failed to download index parts for all timelines. Missing {missing:?}")
}
TenantIndexParts::Present(parts) => Ok(parts),
}
}
/// Retrieves index data from the remote storage for a given timeline.
async fn download_index_part(
conf: &'static PageServerConf,
storage: &GenericRemoteStorage,
sync_id: TenantTimelineId,
) -> Result<IndexPart, DownloadError> {
let index_part_path = conf
.metadata_path(sync_id.timeline_id, sync_id.tenant_id)
.with_file_name(IndexPart::FILE_NAME);
let mut index_part_download = storage
.download_storage_object(None, &index_part_path)
.await?;
let mut index_part_bytes = Vec::new();
io::copy(
&mut index_part_download.download_stream,
&mut index_part_bytes,
)
.await
.with_context(|| {
format!(
"Failed to download an index part into file '{}'",
index_part_path.display()
)
})
.map_err(DownloadError::Other)?;
let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
.with_context(|| {
format!(
"Failed to deserialize index part file into file '{}'",
index_part_path.display()
)
})
.map_err(DownloadError::Other)?;
let missing_files = index_part.missing_files();
if !missing_files.is_empty() {
warn!("Found missing layers in index part for timeline {sync_id}: {missing_files:?}");
}
Ok(index_part)
}
/// Timeline download result, with extra data, needed for downloading.
#[derive(Debug)]
pub(super) enum DownloadedTimeline {
/// Remote timeline data is either absent or corrupt, no download possible.
Abort,
/// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
/// Initial download failed due to some error, the download task is rescheduled for another retry.
FailedAndRescheduled,
/// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
/// Initial download successful.
Successful(SyncData<LayersDownload>),
}
/// Attempts to download all given timeline's layers.
/// Timeline files that already exist locally are skipped during the download, but the local metadata file is
/// updated in the end, if the remote one contains a newer disk_consistent_lsn.
///
/// On an error, bumps the retries count and updates the files to skip with successful downloads, rescheduling the task.
pub(super) async fn download_timeline_layers<'a>(
/// If 'metadata' is given, we will validate that the downloaded file's size matches that
/// in the metadata. (In the future, we might do more cross-checks, like CRC validation)
///
/// Returns the size of the downloaded file.
pub async fn download_layer_file<'a>(
conf: &'static PageServerConf,
storage: &'a GenericRemoteStorage,
sync_queue: &'a SyncQueue,
remote_timeline: Option<&'a RemoteTimeline>,
sync_id: TenantTimelineId,
mut download_data: SyncData<LayersDownload>,
) -> DownloadedTimeline {
let remote_timeline = match remote_timeline {
Some(remote_timeline) => {
if !remote_timeline.awaits_download {
error!("Timeline with sync id {sync_id} is not awaiting download");
return DownloadedTimeline::Abort;
}
remote_timeline
}
None => {
error!("Timeline with sync id {sync_id} is not present in the remote index");
return DownloadedTimeline::Abort;
}
};
let download = &mut download_data.data;
let layers_to_download = remote_timeline
.stored_files()
.iter()
.filter_map(|(layer_path, metadata)| {
if !download.layers_to_skip.contains(layer_path) {
Some((layer_path.to_owned(), metadata.to_owned()))
} else {
None
}
})
.collect::<Vec<_>>();
debug!("Layers to download: {layers_to_download:?}");
info!("Downloading {} timeline layers", layers_to_download.len());
if layers_to_download.is_empty() {
info!("No layers to download after filtering, skipping");
return DownloadedTimeline::Successful(download_data);
}
let mut download_tasks = layers_to_download
.into_iter()
.map(|(layer_destination_path, metadata)| async move {
match layer_destination_path.metadata() {
Ok(m) if m.is_file() => {
// the file exists from earlier round when we failed after renaming it as
// layer_destination_path
let verified = if let Some(expected) = metadata.file_size() {
m.len() == expected
} else {
// behaviour before recording metadata was to accept any existing
true
};
if verified {
debug!(
"Layer already exists locally, skipping download: {}",
layer_destination_path.display()
);
return Ok((layer_destination_path, LayerFileMetadata::new(m.len())))
} else {
// no need to remove it, it will be overwritten by fs::rename
// after successful download
warn!("Downloaded layer exists already but layer file metadata mismatches: {}, metadata {:?}", layer_destination_path.display(), metadata);
}
}
Ok(m) => {
return Err(anyhow::anyhow!("Downloaded layer destination exists but is not a file: {m:?}, target needs to be removed/archived manually: {layer_destination_path:?}"));
}
Err(_) => {
// behave as the file didn't exist
}
}
// Perform a rename inspired by durable_rename from file_utils.c.
// The sequence:
// write(tmp)
// fsync(tmp)
// rename(tmp, new)
// fsync(new)
// fsync(parent)
// For more context about durable_rename check this email from postgres mailing list:
// https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
// If pageserver crashes the temp file will be deleted on startup and re-downloaded.
let temp_file_path =
path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX);
// TODO: this doesn't use the cached fd for some reason?
let mut destination_file =
fs::File::create(&temp_file_path).await.with_context(|| {
format!(
"Failed to create a destination file for layer '{}'",
temp_file_path.display()
)
})?;
let mut layer_download = storage.download_storage_object(None, &layer_destination_path)
.await
.with_context(|| {
format!(
"Failed to initiate the download the layer for {sync_id} into file '{}'",
temp_file_path.display()
)
})?;
let bytes_amount = io::copy(&mut layer_download.download_stream, &mut destination_file)
.await
.with_context(|| {
format!(
"Failed to download the layer for {sync_id} into file '{}'",
temp_file_path.display()
)
})?;
// Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
// A file will not be closed immediately when it goes out of scope if there are any IO operations
// that have not yet completed. To ensure that a file is closed immediately when it is dropped,
// you should call flush before dropping it.
//
// From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because
// we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations.
// But for additional safety let's check/wait for any pending operations.
destination_file.flush().await.with_context(|| {
format!(
"failed to flush source file at {}",
temp_file_path.display()
)
})?;
match metadata.file_size() {
Some(expected) if expected != bytes_amount => {
anyhow::bail!(
"According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
temp_file_path.display()
);
},
Some(_) | None => {
// matches, or upgrading from an earlier IndexPart version
}
}
// not using sync_data because it can lose file size update
destination_file.sync_all().await.with_context(|| {
format!(
"failed to fsync source file at {}",
temp_file_path.display()
)
})?;
drop(destination_file);
fail::fail_point!("remote-storage-download-pre-rename", |_| {
anyhow::bail!("remote-storage-download-pre-rename failpoint triggered")
});
fs::rename(&temp_file_path, &layer_destination_path).await?;
fsync_path(&layer_destination_path).await.with_context(|| {
format!(
"Cannot fsync layer destination path {}",
layer_destination_path.display(),
)
})?;
Ok::<_, anyhow::Error>((layer_destination_path, LayerFileMetadata::new(bytes_amount)))
})
.collect::<FuturesUnordered<_>>();
let mut errors_happened = false;
// keep files we've downloaded to remove them from layers_to_skip if directory fsync fails
let mut undo = HashSet::new();
while let Some(download_result) = download_tasks.next().await {
match download_result {
Ok((downloaded_path, metadata)) => {
undo.insert(downloaded_path.clone());
download.layers_to_skip.insert(downloaded_path.clone());
// what if the key existed already? ignore, because then we would had
// downloaded a partial file, and had to retry
download.gathered_metadata.insert(downloaded_path, metadata);
}
Err(e) => {
errors_happened = true;
error!("Failed to download a layer for timeline {sync_id}: {e:?}");
}
}
}
// fsync timeline directory which is a parent directory for downloaded files
let TenantTimelineId {
tenant_id,
timeline_id,
} = &sync_id;
let timeline_dir = conf.timeline_path(timeline_id, tenant_id);
if let Err(e) = fsync_path(&timeline_dir).await {
error!(
"Cannot fsync parent directory {} error {}",
timeline_dir.display(),
e
);
for item in undo {
download.layers_to_skip.remove(&item);
// intentionally don't clear the gathered_metadata because it exists for fsync_path
// failure on parent directory
}
errors_happened = true;
}
if errors_happened {
debug!("Reenqueuing failed download task for timeline {sync_id}");
download_data.retries += 1;
sync_queue.push(sync_id, SyncTask::Download(download_data));
DownloadedTimeline::FailedAndRescheduled
} else {
info!("Successfully downloaded all layers");
DownloadedTimeline::Successful(download_data)
}
tenant_id: TenantId,
timeline_id: TimelineId,
path: &'a RelativePath,
layer_metadata: &'a LayerFileMetadata,
) -> anyhow::Result<u64> {
download_layer_file_guts(conf, storage, tenant_id, timeline_id, path, layer_metadata).await
}
async fn get_timeline_sync_ids(
storage: &GenericRemoteStorage,
tenant_path: &Path,
async fn download_layer_file_guts<'a>(
conf: &'static PageServerConf,
storage: &'a GenericRemoteStorage,
tenant_id: TenantId,
) -> anyhow::Result<HashSet<TenantTimelineId>> {
let tenant_storage_path = storage.remote_object_id(tenant_path).with_context(|| {
timeline_id: TimelineId,
path: &'a RelativePath,
layer_metadata: &'a LayerFileMetadata,
) -> anyhow::Result<u64> {
const TEMP_DOWNLOAD_EXTENSION: &str = "temp_download";
let timeline_path = conf.timeline_path(&timeline_id, &tenant_id);
let local_path = path.to_local_path(&timeline_path);
let layer_storage_path = storage.remote_object_id(&local_path).with_context(|| {
format!(
"Failed to get the layer storage path for local path '{}'",
local_path.display()
)
})?;
// Perform a rename inspired by durable_rename from file_utils.c.
// The sequence:
// write(tmp)
// fsync(tmp)
// rename(tmp, new)
// fsync(new)
// fsync(parent)
// For more context about durable_rename check this email from postgres mailing list:
// https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
// If pageserver crashes the temp file will be deleted on startup and re-downloaded.
let temp_file_path = path_with_suffix_extension(&local_path, TEMP_DOWNLOAD_EXTENSION);
// TODO: this doesn't use the cached fd for some reason?
let mut destination_file = fs::File::create(&temp_file_path).await.with_context(|| {
format!(
"Failed to create a destination file for layer '{}'",
temp_file_path.display()
)
})?;
let mut download = storage
.download(&layer_storage_path)
.await
.with_context(|| {
format!(
"Failed to open a download stream for layer with remote storage path '{layer_storage_path:?}'"
)
})?;
let bytes_amount = tokio::io::copy(&mut download.download_stream, &mut destination_file).await.with_context(|| {
format!(
"Failed to download layer with remote storage path '{layer_storage_path:?}' into file '{}'", temp_file_path.display()
)
})?;
// Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
// A file will not be closed immediately when it goes out of scope if there are any IO operations
// that have not yet completed. To ensure that a file is closed immediately when it is dropped,
// you should call flush before dropping it.
//
// From the tokio code I see that it waits for pending operations to complete. There shouldt be any because
// we assume that `destination_file` file is fully written. I e there is no pending .write(...).await operations.
// But for additional safety lets check/wait for any pending operations.
destination_file.flush().await.with_context(|| {
format!(
"failed to flush source file at {}",
temp_file_path.display()
)
})?;
match layer_metadata.file_size() {
Some(expected) if expected != bytes_amount => {
anyhow::bail!(
"According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
temp_file_path.display()
);
}
Some(_) | None => {
// matches, or upgrading from an earlier IndexPart version
}
}
// not using sync_data because it can lose file size update
destination_file.sync_all().await.with_context(|| {
format!(
"failed to fsync source file at {}",
temp_file_path.display()
)
})?;
drop(destination_file);
fail::fail_point!("remote-storage-download-pre-rename", |_| {
bail!("remote-storage-download-pre-rename failpoint triggered")
});
fs::rename(&temp_file_path, &local_path).await?;
fsync_path(&local_path)
.await
.with_context(|| format!("Could not fsync layer file {}", local_path.display(),))?;
tracing::info!("download complete: {}", local_path.display());
Ok(bytes_amount)
}
/// List timelines of given tenant in remote storage
pub async fn list_remote_timelines<'a>(
storage: &'a GenericRemoteStorage,
conf: &'static PageServerConf,
tenant_id: TenantId,
) -> anyhow::Result<Vec<(TimelineId, IndexPart)>> {
let tenant_path = conf.timelines_path(&tenant_id);
let tenant_storage_path = storage.remote_object_id(&tenant_path).with_context(|| {
format!(
"Failed to get tenant storage path for local path '{}'",
tenant_path.display()
@@ -444,7 +168,8 @@ async fn get_timeline_sync_ids(
anyhow::bail!("no timelines found on the remote storage")
}
let mut sync_ids = HashSet::new();
let mut timeline_ids = HashSet::new();
let mut part_downloads = FuturesUnordered::new();
for timeline_remote_storage_key in timelines {
let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
@@ -455,241 +180,78 @@ async fn get_timeline_sync_ids(
format!("failed to parse object name into timeline id '{object_name}'")
})?;
sync_ids.insert(TenantTimelineId {
tenant_id,
timeline_id,
});
// list_prefixes returns all files with the prefix. If we haven't seen this timeline ID
// yet, launch a download task for it.
if !timeline_ids.contains(&timeline_id) {
timeline_ids.insert(timeline_id);
let storage_clone = storage.clone();
part_downloads.push(async move {
(
timeline_id,
download_index_part(conf, &storage_clone, tenant_id, timeline_id).await,
)
});
}
}
Ok(sync_ids)
// Wait for all the download tasks to complete.
let mut timeline_parts = Vec::new();
while let Some((timeline_id, part_upload_result)) = part_downloads.next().await {
let index_part = part_upload_result
.with_context(|| format!("Failed to fetch index part for timeline {timeline_id}"))?;
debug!("Successfully fetched index part for timeline {timeline_id}");
timeline_parts.push((timeline_id, index_part));
}
Ok(timeline_parts)
}
async fn fsync_path(path: impl AsRef<Path>) -> Result<(), io::Error> {
fs::File::open(path).await?.sync_all().await
}
pub async fn download_index_part(
conf: &'static PageServerConf,
storage: &GenericRemoteStorage,
tenant_id: TenantId,
timeline_id: TimelineId,
) -> Result<IndexPart, DownloadError> {
async {
let index_part_path = conf
.metadata_path(timeline_id, tenant_id)
.with_file_name(IndexPart::FILE_NAME);
let part_storage_path = storage
.remote_object_id(&index_part_path)
.with_context(|| {
format!(
"Failed to get the index part storage path for local path '{}'",
index_part_path.display()
)
})
.map_err(DownloadError::BadInput)?;
#[cfg(test)]
mod tests {
use std::{
collections::{BTreeSet, HashSet},
num::NonZeroUsize,
path::PathBuf,
};
let mut index_part_download = storage.download(&part_storage_path).await?;
use remote_storage::{LocalFs, RemoteStorage};
use tempfile::tempdir;
use utils::lsn::Lsn;
use crate::{
storage_sync::{
index::RelativePath,
test_utils::{create_local_timeline, dummy_metadata},
},
tenant::harness::{TenantHarness, TIMELINE_ID},
};
use super::*;
#[tokio::test]
async fn download_timeline() -> anyhow::Result<()> {
let harness = TenantHarness::create("download_timeline")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let layer_files = ["a", "b", "layer_to_skip", "layer_to_keep_locally"];
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_owned(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let current_retries = 3;
let metadata = dummy_metadata(Lsn(0x30));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let timeline_upload =
create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
for local_path in timeline_upload.layers_to_upload.keys() {
let remote_path =
local_storage.resolve_in_storage(&storage.remote_object_id(local_path)?)?;
let remote_parent_dir = remote_path.parent().unwrap();
if !remote_parent_dir.exists() {
fs::create_dir_all(&remote_parent_dir).await?;
}
fs::copy(&local_path, &remote_path).await?;
}
let mut read_dir = fs::read_dir(&local_timeline_path).await?;
while let Some(dir_entry) = read_dir.next_entry().await? {
if dir_entry.file_name().to_str() == Some("layer_to_keep_locally") {
continue;
} else {
fs::remove_file(dir_entry.path()).await?;
}
}
let mut remote_timeline = RemoteTimeline::new(metadata.clone());
remote_timeline.awaits_download = true;
remote_timeline.add_timeline_layers(layer_files.iter().map(|layer| {
let layer_path = local_timeline_path.join(layer);
// this could had also been LayerFileMetadata::default(), but since in this test we
// don't do the merge operation done by storage_sync::download_timeline_data, it would
// not be merged back to timeline.
let metadata_from_upload = timeline_upload
.layers_to_upload
.get(&layer_path)
.expect("layer must exist in previously uploaded paths")
.to_owned();
(layer_path, metadata_from_upload)
}));
let download_data = match download_timeline_layers(
harness.conf,
&storage,
&sync_queue,
Some(&remote_timeline),
sync_id,
SyncData::new(
current_retries,
LayersDownload::from_skipped_layers(HashSet::from([
local_timeline_path.join("layer_to_skip")
])),
),
let mut index_part_bytes = Vec::new();
tokio::io::copy(
&mut index_part_download.download_stream,
&mut index_part_bytes,
)
.await
{
DownloadedTimeline::Successful(data) => data,
wrong_result => {
panic!("Expected a successful download for timeline, but got: {wrong_result:?}")
}
};
.with_context(|| {
format!(
"Failed to download an index part into file '{}'",
index_part_path.display()
)
})
.map_err(DownloadError::Other)?;
assert_eq!(
current_retries, download_data.retries,
"On successful download, retries are not expected to change"
);
assert_eq!(
download_data
.data
.layers_to_skip
.into_iter()
.collect::<BTreeSet<_>>(),
layer_files
.iter()
.map(|layer| local_timeline_path.join(layer))
.collect(),
"On successful download, layers to skip should contain all downloaded files and present layers that were skipped"
);
let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
.with_context(|| {
format!(
"Failed to deserialize index part file into file '{}'",
index_part_path.display()
)
})
.map_err(DownloadError::Other)?;
let mut downloaded_files = BTreeSet::new();
let mut read_dir = fs::read_dir(&local_timeline_path).await?;
while let Some(dir_entry) = read_dir.next_entry().await? {
downloaded_files.insert(dir_entry.path());
}
assert_eq!(
downloaded_files,
layer_files
.iter()
.filter(|layer| layer != &&"layer_to_skip")
.map(|layer| local_timeline_path.join(layer))
.collect(),
"On successful download, all layers that were not skipped, should be downloaded"
);
Ok(())
}
#[tokio::test]
async fn download_timeline_negatives() -> anyhow::Result<()> {
let harness = TenantHarness::create("download_timeline_negatives")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_owned(),
harness.conf.workdir.clone(),
)?);
let empty_remote_timeline_download = download_timeline_layers(
harness.conf,
&storage,
&sync_queue,
None,
sync_id,
SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
)
.await;
assert!(
matches!(empty_remote_timeline_download, DownloadedTimeline::Abort),
"Should not allow downloading for empty remote timeline"
);
let not_expecting_download_remote_timeline = RemoteTimeline::new(dummy_metadata(Lsn(5)));
assert!(
!not_expecting_download_remote_timeline.awaits_download,
"Should not expect download for the timeline"
);
let already_downloading_remote_timeline_download = download_timeline_layers(
harness.conf,
&storage,
&sync_queue,
Some(&not_expecting_download_remote_timeline),
sync_id,
SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
)
.await;
assert!(
matches!(
already_downloading_remote_timeline_download,
DownloadedTimeline::Abort,
),
"Should not allow downloading for remote timeline that does not expect it"
);
Ok(())
}
#[tokio::test]
async fn test_download_index_part() -> anyhow::Result<()> {
let harness = TenantHarness::create("test_download_index_part")?;
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_owned(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let metadata = dummy_metadata(Lsn(0x30));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let index_part = IndexPart::new(
HashSet::from([
RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
]),
HashSet::from([RelativePath::new(
&local_timeline_path,
local_timeline_path.join("three"),
)?]),
metadata.disk_consistent_lsn(),
metadata.to_bytes()?,
);
let local_index_part_path = harness
.conf
.metadata_path(sync_id.timeline_id, sync_id.tenant_id)
.with_file_name(IndexPart::FILE_NAME);
let index_part_remote_id = local_storage.remote_object_id(&local_index_part_path)?;
let index_part_local_path = PathBuf::from(index_part_remote_id.to_string());
fs::create_dir_all(index_part_local_path.parent().unwrap()).await?;
fs::write(&index_part_local_path, serde_json::to_vec(&index_part)?).await?;
let downloaded_index_part = download_index_part(harness.conf, &storage, sync_id).await?;
assert_eq!(
downloaded_index_part, index_part,
"Downloaded index part should be the same as the one in storage"
);
Ok(())
Ok(index_part)
}
.await
}
pageserver/src/storage_sync/index.rs
+42 -506
View File
@@ -2,26 +2,18 @@
//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
//! remote timeline layers and its metadata.
use std::ops::{Deref, DerefMut};
use std::{
collections::{HashMap, HashSet},
path::{Path, PathBuf},
sync::Arc,
};
use anyhow::{anyhow, Context, Ok};
use anyhow::{Context, Ok};
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};
use tokio::sync::RwLock;
use tracing::log::warn;
use crate::{config::PageServerConf, tenant::metadata::TimelineMetadata};
use utils::{
id::{TenantId, TenantTimelineId, TimelineId},
lsn::Lsn,
};
use crate::tenant::metadata::TimelineMetadata;
use super::download::TenantIndexParts;
use utils::lsn::Lsn;
/// A part of the filesystem path, that needs a root to become a path again.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
@@ -30,270 +22,23 @@ pub struct RelativePath(String);
impl RelativePath {
/// Attempts to strip off the base from path, producing a relative path or an error.
pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
let path = path.as_ref();
let relative = path.strip_prefix(base).with_context(|| {
pub fn from_local_path(timeline_path: &Path, path: &Path) -> anyhow::Result<RelativePath> {
let relative = path.strip_prefix(timeline_path).with_context(|| {
format!(
"path '{}' is not relative to base '{}'",
path.display(),
base.display()
timeline_path.display()
)
})?;
Ok(RelativePath(relative.to_string_lossy().to_string()))
Ok(Self::from_filename(relative))
}
/// Joins the relative path with the base path.
fn as_path(&self, base: &Path) -> PathBuf {
base.join(&self.0)
}
}
#[derive(Debug, Clone, Default)]
pub struct TenantEntry(HashMap<TimelineId, RemoteTimeline>);
impl TenantEntry {
pub fn has_in_progress_downloads(&self) -> bool {
self.values()
.any(|remote_timeline| remote_timeline.awaits_download)
}
}
impl Deref for TenantEntry {
type Target = HashMap<TimelineId, RemoteTimeline>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for TenantEntry {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl From<HashMap<TimelineId, RemoteTimeline>> for TenantEntry {
fn from(inner: HashMap<TimelineId, RemoteTimeline>) -> Self {
Self(inner)
}
}
/// An index to track tenant files that exist on the remote storage.
#[derive(Debug, Clone, Default)]
pub struct RemoteTimelineIndex {
entries: HashMap<TenantId, TenantEntry>,
}
/// A wrapper to synchronize the access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`].
#[derive(Default)]
pub struct RemoteIndex(Arc<RwLock<RemoteTimelineIndex>>);
impl RemoteIndex {
pub fn from_parts(
conf: &'static PageServerConf,
index_parts: HashMap<TenantId, TenantIndexParts>,
) -> anyhow::Result<Self> {
let mut entries: HashMap<TenantId, TenantEntry> = HashMap::new();
for (tenant_id, index_parts) in index_parts {
match index_parts {
// TODO: should we schedule a retry so it can be recovered? otherwise we can revive it only through detach/attach or pageserver restart
TenantIndexParts::Poisoned { missing, ..} => warn!("skipping tenant_id set up for remote index because the index download has failed for timeline(s): {missing:?}"),
TenantIndexParts::Present(timelines) => {
for (timeline_id, index_part) in timelines {
let timeline_path = conf.timeline_path(&timeline_id, &tenant_id);
let remote_timeline =
RemoteTimeline::from_index_part(&timeline_path, index_part)
.context("Failed to restore remote timeline data from index part")?;
entries
.entry(tenant_id)
.or_default()
.insert(timeline_id, remote_timeline);
}
},
}
}
Ok(Self(Arc::new(RwLock::new(RemoteTimelineIndex { entries }))))
pub fn from_filename(path: &Path) -> RelativePath {
RelativePath(path.to_string_lossy().to_string())
}
pub async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, RemoteTimelineIndex> {
self.0.read().await
}
pub async fn write(&self) -> tokio::sync::RwLockWriteGuard<'_, RemoteTimelineIndex> {
self.0.write().await
}
}
impl Clone for RemoteIndex {
fn clone(&self) -> Self {
Self(Arc::clone(&self.0))
}
}
impl RemoteTimelineIndex {
pub fn timeline_entry(
&self,
TenantTimelineId {
tenant_id,
timeline_id,
}: &TenantTimelineId,
) -> Option<&RemoteTimeline> {
self.entries.get(tenant_id)?.get(timeline_id)
}
pub fn timeline_entry_mut(
&mut self,
TenantTimelineId {
tenant_id,
timeline_id,
}: &TenantTimelineId,
) -> Option<&mut RemoteTimeline> {
self.entries.get_mut(tenant_id)?.get_mut(timeline_id)
}
pub fn add_timeline_entry(
&mut self,
TenantTimelineId {
tenant_id,
timeline_id,
}: TenantTimelineId,
entry: RemoteTimeline,
) {
self.entries
.entry(tenant_id)
.or_default()
.insert(timeline_id, entry);
}
pub fn remove_timeline_entry(
&mut self,
TenantTimelineId {
tenant_id,
timeline_id,
}: TenantTimelineId,
) -> Option<RemoteTimeline> {
self.entries
.entry(tenant_id)
.or_default()
.remove(&timeline_id)
}
pub fn tenant_entry(&self, tenant_id: &TenantId) -> Option<&TenantEntry> {
self.entries.get(tenant_id)
}
pub fn tenant_entry_mut(&mut self, tenant_id: &TenantId) -> Option<&mut TenantEntry> {
self.entries.get_mut(tenant_id)
}
pub fn add_tenant_entry(&mut self, tenant_id: TenantId) -> &mut TenantEntry {
self.entries.entry(tenant_id).or_default()
}
pub fn remove_tenant_entry(&mut self, tenant_id: &TenantId) -> Option<TenantEntry> {
self.entries.remove(tenant_id)
}
pub fn set_awaits_download(
&mut self,
id: &TenantTimelineId,
awaits_download: bool,
) -> anyhow::Result<()> {
self.timeline_entry_mut(id)
.ok_or_else(|| anyhow!("unknown timeline sync {id}"))?
.awaits_download = awaits_download;
Ok(())
}
}
/// Restored index part data about the timeline, stored in the remote index.
#[derive(Debug, Clone)]
pub struct RemoteTimeline {
timeline_layers: HashMap<PathBuf, LayerFileMetadata>,
missing_layers: HashMap<PathBuf, LayerFileMetadata>,
pub metadata: TimelineMetadata,
pub awaits_download: bool,
}
impl RemoteTimeline {
pub fn new(metadata: TimelineMetadata) -> Self {
Self {
timeline_layers: HashMap::default(),
missing_layers: HashMap::default(),
metadata,
awaits_download: false,
}
}
pub fn add_timeline_layers(
&mut self,
new_layers: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
) {
self.timeline_layers.extend(new_layers);
}
pub fn add_upload_failures(
&mut self,
upload_failures: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
) {
self.missing_layers.extend(upload_failures);
}
pub fn remove_layers(&mut self, layers_to_remove: &HashSet<PathBuf>) {
self.timeline_layers
.retain(|layer, _| !layers_to_remove.contains(layer));
self.missing_layers
.retain(|layer, _| !layers_to_remove.contains(layer));
}
/// Lists all layer files in the given remote timeline. Omits the metadata file.
pub fn stored_files(&self) -> &HashMap<PathBuf, LayerFileMetadata> {
&self.timeline_layers
}
/// Combines metadata gathered or verified during downloading needed layer files to metadata on
/// the [`RemoteIndex`], so it can be uploaded later.
pub fn merge_metadata_from_downloaded(
&mut self,
downloaded: &HashMap<PathBuf, LayerFileMetadata>,
) {
downloaded.iter().for_each(|(path, metadata)| {
if let Some(upgraded) = self.timeline_layers.get_mut(path) {
upgraded.merge(metadata);
}
});
}
pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result<Self> {
let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?;
let default_metadata = &IndexLayerMetadata::default();
let find_metadata = |key: &RelativePath| -> LayerFileMetadata {
index_part
.layer_metadata
.get(key)
.unwrap_or(default_metadata)
.into()
};
Ok(Self {
timeline_layers: index_part
.timeline_layers
.iter()
.map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
.collect(),
missing_layers: index_part
.missing_layers
.iter()
.map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
.collect(),
metadata,
awaits_download: false,
})
pub fn to_local_path(&self, timeline_path: &Path) -> PathBuf {
timeline_path.join(&self.0)
}
}
@@ -322,6 +67,10 @@ impl LayerFileMetadata {
}
}
/// This is used to initialize the metadata for remote layers, for which
/// the metadata was missing from the index part file.
pub const MISSING: Self = LayerFileMetadata { file_size: None };
pub fn file_size(&self) -> Option<u64> {
self.file_size
}
@@ -335,7 +84,8 @@ impl LayerFileMetadata {
}
}
/// Part of the remote index, corresponding to a certain timeline.
/// In-memory representation of an `index_part.json` file
///
/// Contains the data about all files in the timeline, present remotely and its metadata.
///
/// This type needs to be backwards and forwards compatible. When changing the fields,
@@ -350,13 +100,10 @@ pub struct IndexPart {
/// Each of the layers present on remote storage.
///
/// Additional metadata can might exist in `layer_metadata`.
timeline_layers: HashSet<RelativePath>,
pub timeline_layers: HashSet<RelativePath>,
/// Currently is not really used in pageserver,
/// present to manually keep track of the layer files that pageserver might never retrieve.
///
/// Such "holes" might appear if any upload task was evicted on an error threshold:
/// the this layer will only be rescheduled for upload on pageserver restart.
/// FIXME: unused field. This should be removed, but that changes the on-disk format,
/// so we need to make sure we're backwards- (and maybe forwards-) compatible
missing_layers: HashSet<RelativePath>,
/// Per layer file metadata, which can be present for a present or missing layer file.
@@ -364,10 +111,12 @@ pub struct IndexPart {
/// Older versions of `IndexPart` will not have this property or have only a part of metadata
/// that latest version stores.
#[serde(default)]
layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
pub layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
// 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata.
// It's duplicated here for convenience.
#[serde_as(as = "DisplayFromStr")]
disk_consistent_lsn: Lsn,
pub disk_consistent_lsn: Lsn,
metadata_bytes: Vec<u8>,
}
@@ -379,63 +128,32 @@ impl IndexPart {
const LATEST_VERSION: usize = 1;
pub const FILE_NAME: &'static str = "index_part.json";
#[cfg(test)]
pub fn new(
timeline_layers: HashSet<RelativePath>,
missing_layers: HashSet<RelativePath>,
layers_and_metadata: HashMap<RelativePath, LayerFileMetadata>,
disk_consistent_lsn: Lsn,
metadata_bytes: Vec<u8>,
) -> Self {
let mut timeline_layers = HashSet::new();
let mut layer_metadata = HashMap::new();
separate_paths_and_metadata(
&layers_and_metadata,
&mut timeline_layers,
&mut layer_metadata,
);
Self {
version: Self::LATEST_VERSION,
timeline_layers,
missing_layers,
layer_metadata: HashMap::default(),
missing_layers: HashSet::new(),
layer_metadata,
disk_consistent_lsn,
metadata_bytes,
}
}
pub fn missing_files(&self) -> &HashSet<RelativePath> {
&self.missing_layers
}
pub fn from_remote_timeline(
timeline_path: &Path,
remote_timeline: RemoteTimeline,
) -> anyhow::Result<Self> {
let metadata_bytes = remote_timeline.metadata.to_bytes()?;
let mut layer_metadata = HashMap::new();
let mut missing_layers = HashSet::new();
separate_paths_and_metadata(
timeline_path,
&remote_timeline.missing_layers,
&mut missing_layers,
&mut layer_metadata,
)
.context("Failed to convert missing layers' paths to relative ones")?;
let mut timeline_layers = HashSet::new();
separate_paths_and_metadata(
timeline_path,
&remote_timeline.timeline_layers,
&mut timeline_layers,
&mut layer_metadata,
)
.context("Failed to convert timeline layers' paths to relative ones")?;
Ok(Self {
version: Self::LATEST_VERSION,
timeline_layers,
missing_layers,
layer_metadata,
disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(),
metadata_bytes,
})
pub fn parse_metadata(&self) -> anyhow::Result<TimelineMetadata> {
TimelineMetadata::from_bytes(&self.metadata_bytes)
}
}
@@ -454,202 +172,20 @@ impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
}
fn separate_paths_and_metadata(
timeline_path: &Path,
input: &HashMap<PathBuf, LayerFileMetadata>,
input: &HashMap<RelativePath, LayerFileMetadata>,
output: &mut HashSet<RelativePath>,
layer_metadata: &mut HashMap<RelativePath, IndexLayerMetadata>,
) -> anyhow::Result<()> {
) {
for (path, metadata) in input {
let rel_path = RelativePath::new(timeline_path, path)?;
let metadata = IndexLayerMetadata::from(metadata);
layer_metadata.insert(rel_path.clone(), metadata);
output.insert(rel_path);
layer_metadata.insert(path.clone(), metadata);
output.insert(path.clone());
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
use crate::DEFAULT_PG_VERSION;
#[test]
fn index_part_conversion() {
let harness = TenantHarness::create("index_part_conversion").unwrap();
let timeline_path = harness.timeline_path(&TIMELINE_ID);
let metadata = TimelineMetadata::new(
Lsn(5).align(),
Some(Lsn(4)),
None,
Lsn(3),
Lsn(2),
Lsn(1),
DEFAULT_PG_VERSION,
);
let remote_timeline = RemoteTimeline {
timeline_layers: HashMap::from([
(timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
(timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
]),
missing_layers: HashMap::from([
(timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
(timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
]),
metadata: metadata.clone(),
awaits_download: false,
};
let index_part = IndexPart::from_remote_timeline(&timeline_path, remote_timeline.clone())
.expect("Correct remote timeline should be convertible to index part");
assert_eq!(
index_part.timeline_layers.iter().collect::<BTreeSet<_>>(),
BTreeSet::from([
&RelativePath("layer_1".to_string()),
&RelativePath("layer_2".to_string())
]),
"Index part should have all remote timeline layers after the conversion"
);
assert_eq!(
index_part.missing_layers.iter().collect::<BTreeSet<_>>(),
BTreeSet::from([
&RelativePath("missing_1".to_string()),
&RelativePath("missing_2".to_string())
]),
"Index part should have all missing remote timeline layers after the conversion"
);
assert_eq!(
index_part.disk_consistent_lsn,
metadata.disk_consistent_lsn(),
"Index part should have disk consistent lsn from the timeline"
);
assert_eq!(
index_part.metadata_bytes,
metadata
.to_bytes()
.expect("Failed to serialize correct metadata into bytes"),
"Index part should have all missing remote timeline layers after the conversion"
);
let restored_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
.expect("Correct index part should be convertible to remote timeline");
let original_metadata = &remote_timeline.metadata;
let restored_metadata = &restored_timeline.metadata;
// we have to compare the metadata this way, since its header is different after creation and restoration,
// but that is now consireded ok.
assert_eq!(
original_metadata.disk_consistent_lsn(),
restored_metadata.disk_consistent_lsn(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
original_metadata.prev_record_lsn(),
restored_metadata.prev_record_lsn(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
original_metadata.ancestor_timeline(),
restored_metadata.ancestor_timeline(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
original_metadata.ancestor_lsn(),
restored_metadata.ancestor_lsn(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
original_metadata.latest_gc_cutoff_lsn(),
restored_metadata.latest_gc_cutoff_lsn(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
original_metadata.initdb_lsn(),
restored_metadata.initdb_lsn(),
"remote timeline -> index part -> remote timeline conversion should not alter metadata"
);
assert_eq!(
remote_timeline.awaits_download, restored_timeline.awaits_download,
"remote timeline -> index part -> remote timeline conversion should not loose download flag"
);
assert_eq!(
remote_timeline
.timeline_layers
.into_iter()
.collect::<BTreeSet<_>>(),
restored_timeline
.timeline_layers
.into_iter()
.collect::<BTreeSet<_>>(),
"remote timeline -> index part -> remote timeline conversion should not loose layer data"
);
assert_eq!(
remote_timeline
.missing_layers
.into_iter()
.collect::<BTreeSet<_>>(),
restored_timeline
.missing_layers
.into_iter()
.collect::<BTreeSet<_>>(),
"remote timeline -> index part -> remote timeline conversion should not loose missing file data"
);
}
#[test]
fn index_part_conversion_negatives() {
let harness = TenantHarness::create("index_part_conversion_negatives").unwrap();
let timeline_path = harness.timeline_path(&TIMELINE_ID);
let metadata = TimelineMetadata::new(
Lsn(5).align(),
Some(Lsn(4)),
None,
Lsn(3),
Lsn(2),
Lsn(1),
DEFAULT_PG_VERSION,
);
let conversion_result = IndexPart::from_remote_timeline(
&timeline_path,
RemoteTimeline {
timeline_layers: HashMap::from([
(PathBuf::from("bad_path"), LayerFileMetadata::new(1)),
(timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
]),
missing_layers: HashMap::from([
(timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
(timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
]),
metadata: metadata.clone(),
awaits_download: false,
},
);
assert!(conversion_result.is_err(), "Should not be able to convert metadata with layer paths that are not in the timeline directory");
let conversion_result = IndexPart::from_remote_timeline(
&timeline_path,
RemoteTimeline {
timeline_layers: HashMap::from([
(timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
(timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
]),
missing_layers: HashMap::from([
(PathBuf::from("bad_path"), LayerFileMetadata::new(3)),
(timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
]),
metadata,
awaits_download: false,
},
);
assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory");
}
#[test]
fn v0_indexpart_is_parsed() {
pageserver/src/storage_sync/upload.rs
+82 -451
View File
@@ -1,479 +1,110 @@
//! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints.
//! Helper functions to upload files to remote storage with a RemoteStorage
use std::{fmt::Debug, path::PathBuf};
use anyhow::Context;
use futures::stream::{FuturesUnordered, StreamExt};
use remote_storage::GenericRemoteStorage;
use anyhow::{bail, Context};
use fail::fail_point;
use std::path::Path;
use tokio::fs;
use tracing::{debug, error, info, warn};
use utils::id::TenantTimelineId;
use super::{
index::{IndexPart, RemoteTimeline},
LayersUpload, SyncData, SyncQueue,
};
use crate::metrics::NO_LAYERS_UPLOAD;
use crate::{config::PageServerConf, storage_sync::SyncTask};
use super::index::IndexPart;
use crate::config::PageServerConf;
use crate::storage_sync::LayerFileMetadata;
use remote_storage::GenericRemoteStorage;
use utils::id::{TenantId, TimelineId};
/// Serializes and uploads the given index part data to the remote storage.
pub(super) async fn upload_index_part(
pub(super) async fn upload_index_part<'a>(
conf: &'static PageServerConf,
storage: &GenericRemoteStorage,
sync_id: TenantTimelineId,
index_part: IndexPart,
storage: &'a GenericRemoteStorage,
tenant_id: TenantId,
timeline_id: TimelineId,
index_part: &'a IndexPart,
) -> anyhow::Result<()> {
let index_part_bytes = serde_json::to_vec(&index_part)
.context("Failed to serialize index part file into bytes")?;
let index_part_size = index_part_bytes.len();
let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
fail_point!("before-upload-index", |_| {
bail!("failpoint before-upload-index")
});
async {
let index_part_bytes = serde_json::to_vec(&index_part)
.context("Failed to serialize index part file into bytes")?;
let index_part_size = index_part_bytes.len();
let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
let index_part_path = conf
.metadata_path(sync_id.timeline_id, sync_id.tenant_id)
.with_file_name(IndexPart::FILE_NAME);
storage
.upload_storage_object(
Box::new(index_part_bytes),
index_part_size,
&index_part_path,
)
.await
.with_context(|| format!("Failed to upload index part for '{sync_id}'"))
}
/// Timeline upload result, with extra data, needed for uploading.
#[derive(Debug)]
pub(super) enum UploadedTimeline {
/// Upload failed due to some error, the upload task is rescheduled for another retry.
FailedAndRescheduled(anyhow::Error),
/// No issues happened during the upload, all task files were put into the remote storage.
Successful(SyncData<LayersUpload>),
let index_part_path = conf
.metadata_path(timeline_id, tenant_id)
.with_file_name(IndexPart::FILE_NAME);
storage
.upload_storage_object(
Box::new(index_part_bytes),
index_part_size,
&index_part_path,
)
.await
.with_context(|| {
format!("Failed to upload index part for '{tenant_id} / {timeline_id}'")
})
}
.await
}
/// Attempts to upload given layer files.
/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
///
/// On an error, bumps the retries count and reschedules the entire task.
pub(super) async fn upload_timeline_layers<'a>(
storage: &'a GenericRemoteStorage,
sync_queue: &SyncQueue,
remote_timeline: Option<&'a RemoteTimeline>,
sync_id: TenantTimelineId,
mut upload_data: SyncData<LayersUpload>,
) -> UploadedTimeline {
let upload = &mut upload_data.data;
let new_upload_lsn = upload
.metadata
.as_ref()
.map(|meta| meta.disk_consistent_lsn());
pub(super) async fn upload_timeline_layer(
storage: &GenericRemoteStorage,
source_path: &Path,
known_metadata: &LayerFileMetadata,
) -> anyhow::Result<()> {
fail_point!("before-upload-layer", |_| {
bail!("failpoint before-upload-layer")
});
async {
let storage_path = storage.remote_object_id(source_path).with_context(|| {
format!(
"Failed to get the layer storage path for local path '{}'",
source_path.display()
)
})?;
let already_uploaded_layers = remote_timeline
.map(|timeline| {
timeline
.stored_files()
.keys()
.cloned()
.collect::<std::collections::HashSet<_>>()
})
.unwrap_or_default();
let source_file = fs::File::open(&source_path).await.with_context(|| {
format!(
"Failed to open a source file for layer '{}'",
source_path.display()
)
})?;
let layers_to_upload = upload
.layers_to_upload
.iter()
.filter_map(|(k, v)| {
if !already_uploaded_layers.contains(k) {
Some((k.to_owned(), v.to_owned()))
} else {
None
}
})
.collect::<Vec<_>>();
if layers_to_upload.is_empty() {
debug!("No layers to upload after filtering, aborting");
NO_LAYERS_UPLOAD
.with_label_values(&[
&sync_id.tenant_id.to_string(),
&sync_id.timeline_id.to_string(),
])
.inc();
return UploadedTimeline::Successful(upload_data);
}
debug!("Layers to upload: {layers_to_upload:?}");
info!(
"Uploading {} timeline layers, new lsn: {new_upload_lsn:?}",
layers_to_upload.len(),
);
let mut upload_tasks = layers_to_upload
.into_iter()
.map(|(source_path, known_metadata)| async move {
let source_file = match fs::File::open(&source_path).await.with_context(|| {
let fs_size = source_file
.metadata()
.await
.with_context(|| {
format!(
"Failed to upen a source file for layer '{}'",
"Failed to get the source file metadata for layer '{}'",
source_path.display()
)
}) {
Ok(file) => file,
Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)),
};
})?
.len();
let fs_size = source_file
.metadata()
.await
.with_context(|| {
format!(
"Failed to get the source file metadata for layer '{}'",
source_path.display()
)
})
.map_err(UploadError::Other)?
.len();
// FIXME: this looks bad
if let Some(metadata_size) = known_metadata.file_size() {
if metadata_size != fs_size {
return Err(UploadError::Other(anyhow::anyhow!(
"File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"
)));
}
} else {
// this is a silly state we would like to avoid
// FIXME: this looks bad
if let Some(metadata_size) = known_metadata.file_size() {
if metadata_size != fs_size {
bail!("File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}");
}
let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))
.map_err(UploadError::Other)?;
match storage
.upload_storage_object(Box::new(source_file), fs_size, &source_path)
.await
.with_context(|| format!("Failed to upload layer file for {sync_id}"))
{
Ok(()) => Ok(source_path),
Err(e) => Err(UploadError::MissingLocalFile(source_path, e)),
}
})
.collect::<FuturesUnordered<_>>();
let mut errors = Vec::new();
while let Some(upload_result) = upload_tasks.next().await {
match upload_result {
Ok(uploaded_path) => {
let metadata = upload
.layers_to_upload
.remove(&uploaded_path)
.expect("metadata should always exist, assuming no double uploads");
upload.uploaded_layers.insert(uploaded_path, metadata);
}
Err(e) => match e {
UploadError::Other(e) => {
error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
errors.push(format!("{e:#}"));
}
UploadError::MissingLocalFile(source_path, e) => {
if source_path.exists() {
error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
errors.push(format!("{e:#}"));
} else {
// We have run the upload sync task, but the file we wanted to upload is gone.
// This is "fine" due the asynchronous nature of the sync loop: it only reacts to events and might need to
// retry the upload tasks, if S3 or network is down: but during this time, pageserver might still operate and
// run compaction/gc tasks, removing redundant files from disk.
// It's not good to pause GC/compaction because of those and we would rather skip such uploads.
//
// Yet absence of such files might also mean that the timeline metadata file was updated (GC moves the Lsn forward, for instance).
// We don't try to read a more recent version, since it could contain `disk_consistent_lsn` that does not have its upload finished yet.
// This will create "missing" layers and make data inconsistent.
// Instead, we only update the metadata when it was submitted in an upload task as a checkpoint result.
upload.layers_to_upload.remove(&source_path);
warn!(
"Missing locally a layer file {} scheduled for upload, skipping",
source_path.display()
);
}
}
},
} else {
// this is a silly state we would like to avoid
}
}
if errors.is_empty() {
info!("Successfully uploaded all layers");
UploadedTimeline::Successful(upload_data)
} else {
debug!("Reenqueuing failed upload task for timeline {sync_id}");
upload_data.retries += 1;
sync_queue.push(sync_id, SyncTask::Upload(upload_data));
UploadedTimeline::FailedAndRescheduled(anyhow::anyhow!(
"Errors appeared during layer uploads: {:?}",
errors
))
}
}
let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))?;
enum UploadError {
MissingLocalFile(PathBuf, anyhow::Error),
Other(anyhow::Error),
}
#[cfg(test)]
mod tests {
use std::{
collections::{BTreeSet, HashSet},
num::NonZeroUsize,
};
use remote_storage::{LocalFs, RemoteStorage};
use tempfile::tempdir;
use utils::lsn::Lsn;
use crate::{
storage_sync::{
index::RelativePath,
test_utils::{create_local_timeline, dummy_metadata},
},
tenant::harness::{TenantHarness, TIMELINE_ID},
};
use super::{upload_index_part, *};
#[tokio::test]
async fn regular_layer_upload() -> anyhow::Result<()> {
let harness = TenantHarness::create("regular_layer_upload")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let layer_files = ["a", "b"];
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_path_buf(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let current_retries = 3;
let metadata = dummy_metadata(Lsn(0x30));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let mut timeline_upload =
create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
timeline_upload.metadata = None;
assert!(
local_storage.list().await?.is_empty(),
"Storage should be empty before any uploads are made"
);
let upload_result = upload_timeline_layers(
&storage,
&sync_queue,
None,
sync_id,
SyncData::new(current_retries, timeline_upload.clone()),
)
.await;
let upload_data = match upload_result {
UploadedTimeline::Successful(upload_data) => upload_data,
wrong_result => {
panic!("Expected a successful upload for timeline, but got: {wrong_result:?}")
}
};
assert_eq!(
current_retries, upload_data.retries,
"On successful upload, retries are not expected to change"
);
let upload = &upload_data.data;
assert!(
upload.layers_to_upload.is_empty(),
"Successful upload should have no layers left to upload"
);
assert_eq!(
upload
.uploaded_layers
.keys()
.cloned()
.collect::<BTreeSet<_>>(),
layer_files
.iter()
.map(|layer_file| local_timeline_path.join(layer_file))
.collect(),
"Successful upload should have all layers uploaded"
);
assert_eq!(
upload.metadata, None,
"Successful upload without metadata should not have it returned either"
);
let storage_files = local_storage.list().await?;
assert_eq!(
storage_files.len(),
layer_files.len(),
"All layers should be uploaded"
);
assert_eq!(
storage_files
.into_iter()
.map(|storage_path| local_storage.local_path(&storage_path))
.collect::<anyhow::Result<BTreeSet<_>>>()?,
layer_files
.into_iter()
.map(|file| local_timeline_path.join(file))
.collect(),
"Uploaded files should match with the local ones"
);
Ok(())
}
// Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario.
#[tokio::test]
async fn layer_upload_after_local_fs_update() -> anyhow::Result<()> {
let harness = TenantHarness::create("layer_upload_after_local_fs_update")?;
let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let layer_files = ["a1", "b1"];
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_owned(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let current_retries = 5;
let metadata = dummy_metadata(Lsn(0x40));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let layers_to_upload = {
let mut layers = layer_files.to_vec();
layers.push("layer_to_remove");
layers
};
let timeline_upload =
create_local_timeline(&harness, TIMELINE_ID, &layers_to_upload, metadata.clone())
.await?;
assert!(
local_storage.list().await?.is_empty(),
"Storage should be empty before any uploads are made"
);
fs::remove_file(local_timeline_path.join("layer_to_remove")).await?;
let upload_result = upload_timeline_layers(
&storage,
&sync_queue,
None,
sync_id,
SyncData::new(current_retries, timeline_upload.clone()),
)
.await;
let upload_data = match upload_result {
UploadedTimeline::Successful(upload_data) => upload_data,
wrong_result => panic!(
"Expected a successful after local fs upload for timeline, but got: {wrong_result:?}"
),
};
assert_eq!(
current_retries, upload_data.retries,
"On successful upload, retries are not expected to change"
);
let upload = &upload_data.data;
assert!(
upload.layers_to_upload.is_empty(),
"Successful upload should have no layers left to upload, even those that were removed from the local fs"
);
assert_eq!(
upload
.uploaded_layers
.keys()
.cloned()
.collect::<BTreeSet<_>>(),
layer_files
.iter()
.map(|layer_file| local_timeline_path.join(layer_file))
.collect(),
"Successful upload should have all layers uploaded"
);
assert_eq!(
upload.metadata,
Some(metadata),
"Successful upload should not change its metadata"
);
let storage_files = local_storage.list().await?;
assert_eq!(
storage_files.len(),
layer_files.len(),
"All layers should be uploaded"
);
assert_eq!(
storage_files
.into_iter()
.map(|storage_path| local_storage.local_path(&storage_path))
.collect::<anyhow::Result<BTreeSet<_>>>()?,
layer_files
.into_iter()
.map(|file| local_timeline_path.join(file))
.collect(),
"Uploaded files should match with the local ones"
);
Ok(())
}
#[tokio::test]
async fn test_upload_index_part() -> anyhow::Result<()> {
let harness = TenantHarness::create("test_upload_index_part")?;
let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
let storage = GenericRemoteStorage::new(LocalFs::new(
tempdir()?.path().to_owned(),
harness.conf.workdir.clone(),
)?);
let local_storage = storage.as_local().unwrap();
let metadata = dummy_metadata(Lsn(0x40));
let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
let index_part = IndexPart::new(
HashSet::from([
RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
]),
HashSet::from([RelativePath::new(
&local_timeline_path,
local_timeline_path.join("three"),
)?]),
metadata.disk_consistent_lsn(),
metadata.to_bytes()?,
);
assert!(
local_storage.list().await?.is_empty(),
"Storage should be empty before any uploads are made"
);
upload_index_part(harness.conf, &storage, sync_id, index_part.clone()).await?;
let storage_files = local_storage.list().await?;
assert_eq!(
storage_files.len(),
1,
"Should have only the index part file uploaded"
);
let index_part_path = storage_files.first().unwrap();
assert_eq!(
index_part_path.object_name(),
Some(IndexPart::FILE_NAME),
"Remote index part should have the correct name"
);
let remote_index_part: IndexPart = serde_json::from_slice(
&fs::read(local_storage.resolve_in_storage(index_part_path)?).await?,
)?;
assert_eq!(
index_part, remote_index_part,
"Remote index part should match the local one"
);
storage
.upload(Box::new(source_file), fs_size, &storage_path, None)
.await
.with_context(|| {
format!(
"Failed to upload a layer from local path '{}'",
source_path.display()
)
})?;
Ok(())
}
.await
}
pageserver/src/task_mgr.rs
+2 -2
View File
@@ -197,8 +197,8 @@ pub enum TaskKind {
// Task that flushes frozen in-memory layers to disk
LayerFlushTask,
// Task that manages the remote upload queue
StorageSync,
// Task that uploads a file to remote storage
RemoteUploadTask,
// task that handles the initial downloading of all tenants
InitialLoad,
pageserver/src/tenant.rs
+593 -148
View File
File diff suppressed because it is too large Load Diff
pageserver/src/tenant/metadata.rs
+20
View File
@@ -264,6 +264,26 @@ pub fn save_metadata(
Ok(())
}
pub fn load_metadata(
conf: &'static PageServerConf,
timeline_id: TimelineId,
tenant_id: TenantId,
) -> anyhow::Result<TimelineMetadata> {
let metadata_path = conf.metadata_path(timeline_id, tenant_id);
let metadata_bytes = std::fs::read(&metadata_path).with_context(|| {
format!(
"Failed to read metadata bytes from path {}",
metadata_path.display()
)
})?;
TimelineMetadata::from_bytes(&metadata_bytes).with_context(|| {
format!(
"Failed to parse metadata bytes from path {}",
metadata_path.display()
)
})
}
#[cfg(test)]
mod tests {
use super::*;
pageserver/src/tenant/timeline.rs
+296 -59
View File
@@ -6,6 +6,7 @@ use fail::fail_point;
use itertools::Itertools;
use once_cell::sync::OnceCell;
use pageserver_api::models::TimelineState;
use remote_storage::DownloadError;
use tokio::sync::watch;
use tokio::task::spawn_blocking;
use tracing::*;
@@ -14,11 +15,13 @@ use std::cmp::{max, min, Ordering};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::ops::{Deref, Range};
use std::path::PathBuf;
use std::sync::atomic::{self, AtomicBool, AtomicI64, Ordering as AtomicOrdering};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering};
use std::sync::{Arc, Mutex, MutexGuard, RwLock};
use std::time::{Duration, Instant, SystemTime};
use crate::storage_sync::index::{IndexPart, RelativePath};
use crate::storage_sync::RemoteTimelineClient;
use crate::tenant::{
delta_layer::{DeltaLayer, DeltaLayerWriter},
ephemeral_file::is_ephemeral_file,
@@ -56,10 +59,7 @@ use crate::walreceiver::{is_etcd_client_initialized, spawn_connection_manager_ta
use crate::walredo::WalRedoManager;
use crate::CheckpointConfig;
use crate::ZERO_PAGE;
use crate::{
page_cache,
storage_sync::{self, index::LayerFileMetadata},
};
use crate::{page_cache, storage_sync::index::LayerFileMetadata};
pub struct Timeline {
conf: &'static PageServerConf,
@@ -79,6 +79,13 @@ pub struct Timeline {
// WAL redo manager
walredo_mgr: Arc<dyn WalRedoManager + Sync + Send>,
/// Remote storage client.
///
/// If Some, use it to upload all newly created layers to the remote storage,
/// and keep remote metadata file in sync. In the future, also use it to download
/// layer files on-demand.
pub remote_client: Option<Arc<RemoteTimelineClient>>,
// What page versions do we hold in the repository? If we get a
// request > last_record_lsn, we need to wait until we receive all
// the WAL up to the request. The SeqWait provides functions for
@@ -111,9 +118,6 @@ pub struct Timeline {
// Metrics
metrics: TimelineMetrics,
/// If `true`, will backup its files that appear after each checkpointing to the remote storage.
upload_layers: AtomicBool,
/// Ensures layers aren't frozen by checkpointer between
/// [`Timeline::get_layer_for_write`] and layer reads.
/// Locked automatically by [`TimelineWriter`] and checkpointer.
@@ -400,6 +404,14 @@ impl Timeline {
self.disk_consistent_lsn.load()
}
pub fn get_remote_consistent_lsn(&self) -> Option<Lsn> {
if let Some(remote_client) = &self.remote_client {
remote_client.last_uploaded_consistent_lsn()
} else {
None
}
}
/// Get the physical size of the timeline at the latest LSN
pub fn get_physical_size(&self) -> u64 {
self.metrics.current_physical_size_gauge.get()
@@ -546,15 +558,10 @@ impl Timeline {
// 2. Create new image layers for partitions that have been modified
// "enough".
let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?;
if !layer_paths_to_upload.is_empty()
&& self.upload_layers.load(atomic::Ordering::Relaxed)
{
storage_sync::schedule_layer_upload(
self.tenant_id,
self.timeline_id,
layer_paths_to_upload,
None,
);
if let Some(remote_client) = &self.remote_client {
for (path, layer_metadata) in layer_paths_to_upload {
remote_client.schedule_layer_file_upload(&path, &layer_metadata)?;
}
}
// 3. Compact
@@ -719,7 +726,7 @@ impl Timeline {
timeline_id: TimelineId,
tenant_id: TenantId,
walredo_mgr: Arc<dyn WalRedoManager + Send + Sync>,
upload_layers: bool,
remote_client: Option<RemoteTimelineClient>,
pg_version: u32,
) -> Self {
let disk_consistent_lsn = metadata.disk_consistent_lsn();
@@ -738,6 +745,8 @@ impl Timeline {
walredo_mgr,
remote_client: remote_client.map(Arc::new),
// initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'.
last_record_lsn: SeqWait::new(RecordLsn {
last: disk_consistent_lsn,
@@ -753,8 +762,6 @@ impl Timeline {
metrics: TimelineMetrics::new(&tenant_id, &timeline_id),
upload_layers: AtomicBool::new(upload_layers),
flush_loop_started: Mutex::new(false),
layer_flush_start_tx,
@@ -884,7 +891,7 @@ impl Timeline {
imgfilename, self.timeline_id, disk_consistent_lsn
);
rename_to_backup(direntry.path())?;
rename_to_backup(&direntry.path())?;
continue;
}
@@ -908,7 +915,7 @@ impl Timeline {
deltafilename, self.timeline_id, disk_consistent_lsn
);
rename_to_backup(direntry.path())?;
rename_to_backup(&direntry.path())?;
continue;
}
@@ -945,6 +952,253 @@ impl Timeline {
Ok(())
}
async fn download_missing(
&self,
index_part: &IndexPart,
remote_client: &RemoteTimelineClient,
mut local_filenames: HashSet<PathBuf>,
first_save: bool,
) -> anyhow::Result<(HashSet<PathBuf>, TimelineMetadata)> {
let remote_metadata = index_part.parse_metadata().with_context(|| {
format!(
"Failed to parse metadata file from remote storage for tenant {}",
self.tenant_id
)
})?;
let remote_consistent_lsn = remote_metadata.disk_consistent_lsn();
let mut remote_filenames: HashSet<PathBuf> = HashSet::new();
for fname in index_part.timeline_layers.iter() {
remote_filenames.insert(fname.to_local_path(&PathBuf::from("")));
}
// Are there any local files that exist, with a size that doesn't match
// with the size stored in the remote index file?
local_filenames.retain(|path| {
let layer_metadata = index_part
.layer_metadata
.get(&RelativePath::from_filename(path))
.map(LayerFileMetadata::from)
.unwrap_or(LayerFileMetadata::MISSING);
if let Some(remote_size) = layer_metadata.file_size() {
let local_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id).join(&path);
match local_path.metadata() {
Ok(metadata) => {
let local_size = metadata.len();
if local_size != remote_size {
warn!("removing local file \"{}\" because it has unexpected length {}; length in remote index is {}",
path.display(),
local_size,
remote_size);
if let Err(err) = rename_to_backup(&local_path) {
error!("could not rename file \"{}\": {:?}",
local_path.display(), err);
}
false
} else {
true
}
}
Err(err) => {
error!("could not get size of local file \"{}\": {:?}", path.display(), err);
true
}
}
} else {
true
}
});
// Are we missing some files that are present in remote storage?
// Create RemoteLayers to represent them, and add to the layer map
// TODO Downloading many files this way is not efficient.
// Better to use FuturesUnordered. Maybe keep as is because:
// a) inplace download is a throw-away code, on-demand patch doesnt need that
// b) typical case now is that there is nothing to sync, this downloads a lot
// 1) if there was another pageserver that came and generated new files
// 2) during attach of a timeline with big history which we currently do not do
for path in remote_filenames.difference(&local_filenames) {
let fname = path.to_str().unwrap();
info!("remote layer file {fname} does not exist locally");
let layer_metadata = index_part
.layer_metadata
.get(&RelativePath::from_filename(path))
.map(LayerFileMetadata::from)
.unwrap_or(LayerFileMetadata::MISSING);
if let Some(imgfilename) = ImageFileName::parse_str(fname) {
if imgfilename.lsn > remote_consistent_lsn {
warn!(
"found future image layer {} on timeline {} remote_consistent_lsn is {}",
imgfilename, self.timeline_id, remote_consistent_lsn
);
continue;
}
remote_client
.download_layer_file(
&RelativePath::from_local_path(
&self.conf.timeline_path(&self.timeline_id, &self.tenant_id),
path,
)?,
&layer_metadata,
)
.await?;
let image_layer =
ImageLayer::new(self.conf, self.timeline_id, self.tenant_id, &imgfilename);
// FIXME: when to update physical size?
self.layers
.write()
.unwrap()
.insert_historic(Arc::new(image_layer));
} else if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
// Create a DeltaLayer struct for each delta file.
// The end-LSN is exclusive, while disk_consistent_lsn is
// inclusive. For example, if disk_consistent_lsn is 100, it is
// OK for a delta layer to have end LSN 101, but if the end LSN
// is 102, then it might not have been fully flushed to disk
// before crash.
if deltafilename.lsn_range.end > remote_consistent_lsn + 1 {
warn!(
"found future delta layer {} on timeline {} remote_consistent_lsn is {}",
deltafilename, self.timeline_id, remote_consistent_lsn
);
continue;
}
let delta_layer =
DeltaLayer::new(self.conf, self.timeline_id, self.tenant_id, &deltafilename);
// FIXME: when to update physical size?
self.layers
.write()
.unwrap()
.insert_historic(Arc::new(delta_layer));
} else {
bail!("unexpected layer filename in remote storage: {}", fname);
}
}
// Save the metadata file to local disk.
// Do this last, so that if we crash in-between, we won't think that the
// local state is valid.
// TODO we should not blindly overwrite local metadata with remote one
// consider the case when checkpoint came, we updated local metadata and started upload task
// but then pageserver crashes, then on start we'll load new metadata, and then reset it to the state of
// remote one. But current layermap will have layers from the old metadata which is inconsistent.
// if we treat remote as source of truth we need to completely syc with it, i e delete local files which are
// missing on the remote which will add extra work, wal for these layers needs to be reingested
//
// In other words we update local metadata only when remote one has greater disk_consistent_lsn
// FIXME we should at least keep highest disk_consistent_lsn. Something else we need to merge?
save_metadata(
self.conf,
self.timeline_id,
self.tenant_id,
&remote_metadata,
first_save,
)?;
// now these are local only filenames
let local_only_filenames = local_filenames
.difference(&remote_filenames)
.cloned()
.collect();
Ok((local_only_filenames, remote_metadata))
}
///
/// This function will synchronize local data with what we have in remote storage.
/// 1. It will download missing layer files.
/// 2. It will update local metadata if remote one has greater `disk_consistent_lsn`.
/// 3. It will upload files that are missing on the remote
/// 4. It will update index file on the remote accordingly
/// TODO may be a bit cleaner to do things based on populated remote client,
/// and then do things based on its upload_queue.latest_files
///
/// This is used during tenant attach. The layer map must have been loaded
/// with local filesystem contents already.
///
/// The caller can provide IndexPart if it has it already. If it's None,
/// this function will download it.
///
#[instrument(skip(self, index_part), fields(timeline_id = %self.timeline_id))]
pub async fn reconcile_with_remote(
&self,
local_metadata: TimelineMetadata,
index_part: Option<&IndexPart>,
first_save: bool,
) -> anyhow::Result<()> {
info!("starting");
let remote_client = self
.remote_client
.as_ref()
.ok_or_else(|| anyhow!("cannot download without remote storage"))?;
// Build a map of local layers for quick lookups
let mut local_filenames: HashSet<PathBuf> = HashSet::new();
for layer in self.layers.read().unwrap().iter_historic_layers() {
local_filenames.insert(layer.filename());
}
// If the caller supplied an IndexPart, use it. Otherwise download it from remote storage.
// In case there is no such file on remote storage (which is possible if pageserver
// was interrupted before index file was uploaded).
// FIXME One way around it would be not considering timeline as initialized before upload finishes.
// So if there is no index file on the remote we consider all locally existing files as local only
// and schedule their upload
// FIXME looks messy, needs cleanup
let (local_only_filenames, up_to_date_metadata) = if let Some(index_part) = index_part {
let local_only_filenames = self
.download_missing(index_part, remote_client, local_filenames, first_save)
.await?;
remote_client.init_upload_queue(index_part)?;
local_only_filenames
} else {
match remote_client.download_index_file().await {
Ok(index_part) => {
let (local_only_filenames, up_to_date_metadata) = self
.download_missing(&index_part, remote_client, local_filenames, first_save)
.await?;
remote_client.init_upload_queue(&index_part)?;
(local_only_filenames, up_to_date_metadata)
}
Err(DownloadError::NotFound) => {
info!("no index file was found on the remote");
remote_client.init_upload_queue_empty(&local_metadata)?;
(local_filenames, local_metadata)
}
Err(e) => return Err(anyhow::anyhow!(e)),
}
};
// TODO what to do with physical size?
// Are there local files that don't exist remotely? Schedule uploads for them
for fname in &local_only_filenames {
let sz = fname.metadata()?.len();
info!("scheduling {} for upload", fname.display());
remote_client.schedule_layer_file_upload(fname, &LayerFileMetadata::new(sz))?;
}
if !local_only_filenames.is_empty() {
// FIXME this we should merge local and remote metadata, at least remote_consistent_lsn
// see comment in download_missing
remote_client.schedule_index_upload(Some(&up_to_date_metadata))?;
}
info!("Done");
Ok(())
}
pub(super) fn layer_removal_guard(&self) -> anyhow::Result<MutexGuard<()>> {
self.layer_removal_cs
.try_lock()
@@ -1506,13 +1760,11 @@ impl Timeline {
false,
)?;
if self.can_upload_layers() {
storage_sync::schedule_layer_upload(
self.tenant_id,
self.timeline_id,
layer_paths_to_upload,
Some(metadata),
);
if let Some(remote_client) = &self.remote_client {
for (path, layer_metadata) in layer_paths_to_upload {
remote_client.schedule_layer_file_upload(&path, &layer_metadata)?;
}
remote_client.schedule_index_upload(Some(&metadata))?;
}
Ok(())
@@ -1971,32 +2223,26 @@ impl Timeline {
// Now that we have reshuffled the data to set of new delta layers, we can
// delete the old ones
let mut layer_paths_do_delete = HashSet::with_capacity(deltas_to_compact.len());
let mut layer_paths_to_delete = Vec::with_capacity(deltas_to_compact.len());
drop(all_keys_iter);
for l in deltas_to_compact {
if let Some(path) = l.local_path() {
self.metrics
.current_physical_size_gauge
.sub(path.metadata()?.len());
layer_paths_do_delete.insert(path);
layer_paths_to_delete.push(path);
}
l.delete()?;
layers.remove_historic(l);
}
drop(layers);
if self.can_upload_layers() {
storage_sync::schedule_layer_upload(
self.tenant_id,
self.timeline_id,
new_layer_paths,
None,
);
storage_sync::schedule_layer_delete(
self.tenant_id,
self.timeline_id,
layer_paths_do_delete,
);
// Also schedule the deletions in remote storage
if let Some(remote_client) = &self.remote_client {
// FIXME: This also uploads new index file. If
// flush_frozen_layer() is doing this at the same time, do
// we have a problem?
remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
}
Ok(())
@@ -2247,13 +2493,13 @@ impl Timeline {
// Actually delete the layers from disk and remove them from the map.
// (couldn't do this in the loop above, because you cannot modify a collection
// while iterating it. BTreeMap::retain() would be another option)
let mut layer_paths_to_delete = HashSet::with_capacity(layers_to_remove.len());
let mut layer_paths_to_delete = Vec::with_capacity(layers_to_remove.len());
for doomed_layer in layers_to_remove {
if let Some(path) = doomed_layer.local_path() {
self.metrics
.current_physical_size_gauge
.sub(path.metadata()?.len());
layer_paths_to_delete.insert(path);
layer_paths_to_delete.push(path);
}
doomed_layer.delete()?;
layers.remove_historic(doomed_layer);
@@ -2269,12 +2515,8 @@ impl Timeline {
fail_point!("after-timeline-gc-removed-layers");
}
if self.can_upload_layers() {
storage_sync::schedule_layer_delete(
self.tenant_id,
self.timeline_id,
layer_paths_to_delete,
);
if let Some(remote_client) = &self.remote_client {
remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
}
result.elapsed = now.elapsed()?;
@@ -2355,11 +2597,6 @@ impl Timeline {
}
}
}
fn can_upload_layers(&self) -> bool {
self.upload_layers.load(atomic::Ordering::Relaxed)
&& self.current_state() != TimelineState::Broken
}
}
/// Helper function for get_reconstruct_data() to add the path of layers traversed
@@ -2437,12 +2674,12 @@ impl<'a> TimelineWriter<'a> {
/// Add a suffix to a layer file's name: .{num}.old
/// Uses the first available num (starts at 0)
fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> {
fn rename_to_backup(path: &Path) -> anyhow::Result<()> {
let filename = path
.file_name()
.ok_or_else(|| anyhow!("Path {} don't have a file name", path.display()))?
.to_string_lossy();
let mut new_path = path.clone();
let mut new_path = path.to_owned();
for i in 0u32.. {
new_path.set_file_name(format!("{}.{}.old", filename, i));
pageserver/src/tenant_mgr.rs
+104 -513
View File
@@ -1,7 +1,7 @@
//! This module acts as a switchboard to access different repositories managed by this
//! page server.
use std::collections::{hash_map, HashMap};
use std::collections::hash_map;
use std::ffi::OsStr;
use std::fs;
use std::path::{Path, PathBuf};
@@ -12,14 +12,9 @@ use tracing::*;
use remote_storage::GenericRemoteStorage;
use crate::config::{PageServerConf, METADATA_FILE_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
use crate::http::models::TenantInfo;
use crate::storage_sync::index::{LayerFileMetadata, RemoteIndex, RemoteTimelineIndex};
use crate::storage_sync::{self, LocalTimelineInitStatus, SyncStartupData, TimelineLocalFiles};
use crate::config::PageServerConf;
use crate::task_mgr::{self, TaskKind};
use crate::tenant::{
ephemeral_file::is_ephemeral_file, metadata::TimelineMetadata, Tenant, TenantState,
};
use crate::tenant::{Tenant, TenantState};
use crate::tenant_config::TenantConfOpt;
use crate::walredo::PostgresRedoManager;
use crate::TEMP_FILE_SUFFIX;
@@ -59,153 +54,79 @@ mod tenants_state {
pub fn init_tenant_mgr(
conf: &'static PageServerConf,
remote_storage: Option<GenericRemoteStorage>,
) -> anyhow::Result<RemoteIndex> {
) -> anyhow::Result<()> {
let _entered = info_span!("init_tenant_mgr").entered();
let local_tenant_files = local_tenant_timeline_files(conf)
.context("Failed to collect local tenant timeline files")?;
let (remote_index, tenants_to_attach) = if let Some(storage) = remote_storage {
let storage_config = conf
.remote_storage_config
.as_ref()
.expect("remote storage without config");
let mut broken_tenants = HashMap::new();
let mut ready_tenants = HashMap::new();
for (tenant_id, tenant_attach_data) in local_tenant_files.into_iter() {
match tenant_attach_data {
TenantAttachData::Ready(t) => {
ready_tenants.insert(tenant_id, t);
}
TenantAttachData::Broken(e) => {
broken_tenants.insert(tenant_id, TenantAttachData::Broken(e));
// Scan local filesystem for attached tenants
let mut number_of_tenants = 0;
let tenants_dir = conf.tenants_path();
for dir_entry in std::fs::read_dir(&tenants_dir)
.with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
{
match &dir_entry {
Ok(dir_entry) => {
let tenant_dir_path = dir_entry.path();
if crate::is_temporary(&tenant_dir_path) {
info!(
"Found temporary tenant directory, removing: {}",
tenant_dir_path.display()
);
if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) {
error!(
"Failed to remove temporary directory '{}': {:?}",
tenant_dir_path.display(),
e
);
}
} else {
number_of_tenants += 1;
if let Err(e) =
load_local_tenant(conf, &tenant_dir_path, remote_storage.clone())
{
error!(
"Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
tenants_dir.display(),
dir_entry,
e
);
}
}
}
}
let SyncStartupData {
remote_index,
local_timeline_init_statuses,
} = storage_sync::spawn_storage_sync_task(
conf,
ready_tenants,
storage,
storage_config.max_concurrent_syncs,
storage_config.max_sync_errors,
)
.context("Failed to spawn the storage sync thread")?;
let n = local_timeline_init_statuses.0.len();
let mut synced_timelines = local_timeline_init_statuses.0.into_iter().fold(
HashMap::<TenantId, TenantAttachData>::with_capacity(n),
|mut new_values, (tenant_id, old_values)| {
let new_timeline_values = new_values
.entry(tenant_id)
.or_insert_with(|| TenantAttachData::Ready(HashMap::new()));
if let TenantAttachData::Ready(t) = new_timeline_values {
for (timeline_id, old_value) in old_values {
if let LocalTimelineInitStatus::LocallyComplete(metadata) = old_value {
t.insert(timeline_id, TimelineLocalFiles::ready(metadata));
}
}
}
new_values
},
);
synced_timelines.extend(broken_tenants);
(remote_index, synced_timelines)
} else {
info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled");
(RemoteIndex::default(), local_tenant_files)
};
attach_local_tenants(conf, &remote_index, tenants_to_attach);
Ok(remote_index)
}
/// Reads local files to load tenants and their timelines given into pageserver's memory.
/// Ignores other timelines that might be present for tenant, but were not passed as a parameter.
/// Attempts to load as many entites as possible: if a certain timeline fails during the load, the tenant is marked as "Broken",
/// and the load continues.
///
/// For successful tenant attach, it first has to have a `timelines/` subdirectory and a tenant config file that's loaded into memory successfully.
/// If either of the conditions fails, the tenant will be added to memory with [`TenantState::Broken`] state, otherwise we start to load its timelines.
/// Alternatively, tenant is considered loaded successfully, if it's already in pageserver's memory (i.e. was loaded already before).
///
/// Attach happens on startup and sucessful timeline downloads
/// (some subset of timeline files, always including its metadata, after which the new one needs to be registered).
pub fn attach_local_tenants(
conf: &'static PageServerConf,
remote_index: &RemoteIndex,
tenants_to_attach: HashMap<TenantId, TenantAttachData>,
) {
let _entered = info_span!("attach_local_tenants").entered();
let number_of_tenants = tenants_to_attach.len();
for (tenant_id, local_timelines) in tenants_to_attach {
let mut tenants_accessor = tenants_state::write_tenants();
let tenant = match tenants_accessor.entry(tenant_id) {
hash_map::Entry::Occupied(o) => {
info!("Tenant {tenant_id} was found in pageserver's memory");
Arc::clone(o.get())
}
hash_map::Entry::Vacant(v) => {
info!("Tenant {tenant_id} was not found in pageserver's memory, loading it");
let tenant = Arc::new(Tenant::new(
conf,
TenantConfOpt::default(),
Arc::new(PostgresRedoManager::new(conf, tenant_id)),
tenant_id,
remote_index.clone(),
conf.remote_storage_config.is_some(),
));
match local_timelines {
TenantAttachData::Broken(_) => {
tenant.set_state(TenantState::Broken);
}
TenantAttachData::Ready(_) => {
match Tenant::load_tenant_config(conf, tenant_id) {
Ok(tenant_conf) => {
tenant.update_tenant_config(tenant_conf);
tenant.activate(false);
}
Err(e) => {
error!("Failed to read config for tenant {tenant_id}, disabling tenant: {e:?}");
tenant.set_state(TenantState::Broken);
}
};
}
}
v.insert(Arc::clone(&tenant));
tenant
}
};
drop(tenants_accessor);
match local_timelines {
TenantAttachData::Broken(e) => warn!("{}", e),
TenantAttachData::Ready(ref timelines) => {
info!("Attaching {} timelines for {tenant_id}", timelines.len());
debug!("Timelines to attach: {local_timelines:?}");
let has_timelines = !timelines.is_empty();
let timelines_to_attach = timelines
.iter()
.map(|(&k, v)| (k, v.metadata().to_owned()))
.collect();
match tenant.init_attach_timelines(timelines_to_attach) {
Ok(()) => {
info!("successfully loaded local timelines for tenant {tenant_id}");
tenant.activate(has_timelines);
}
Err(e) => {
error!("Failed to attach tenant timelines: {e:?}");
tenant.set_state(TenantState::Broken);
}
}
Err(e) => {
// On error, print it, but continue with the other tenants. If we error out
// here, the pageserver startup fails altogether, causing outage for *all*
// tenants. That seems worse.
error!(
"Failed to list tenants dir entry {:?} in directory {}, reason: {:?}",
dir_entry,
tenants_dir.display(),
e,
);
}
}
}
info!("Processed {number_of_tenants} local tenants during attach")
info!("Processed {number_of_tenants} local tenants at startup");
Ok(())
}
fn load_local_tenant(
conf: &'static PageServerConf,
tenant_path: &Path,
remote_storage: Option<GenericRemoteStorage>,
) -> anyhow::Result<()> {
let tenant_id = tenant_path
.file_name()
.and_then(OsStr::to_str)
.unwrap_or_default()
.parse::<TenantId>()
.context("Could not parse tenant id out of the tenant dir name")?;
// Start loading the tenant into memory. It will initially be in Loading
// state.
let tenant = Tenant::spawn_load(conf, tenant_id, remote_storage)?;
tenants_state::write_tenants().insert(tenant_id, tenant);
Ok(())
}
///
@@ -376,7 +297,7 @@ pub fn create_tenant(
conf: &'static PageServerConf,
tenant_conf: TenantConfOpt,
tenant_id: TenantId,
remote_index: RemoteIndex,
remote_storage: Option<&GenericRemoteStorage>,
) -> anyhow::Result<Option<TenantId>> {
match tenants_state::write_tenants().entry(tenant_id) {
hash_map::Entry::Occupied(_) => {
@@ -386,15 +307,17 @@ pub fn create_tenant(
hash_map::Entry::Vacant(v) => {
let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id));
create_tenant_files(conf, tenant_conf, tenant_id)?;
// TODO review, created in active state immediately, is it correctly?
let tenant = Arc::new(Tenant::new(
TenantState::Active {
background_jobs_running: true,
},
conf,
tenant_conf,
wal_redo_manager,
tenant_id,
remote_index,
conf.remote_storage_config.is_some(),
remote_storage.cloned(),
));
tenant.activate(false);
v.insert(tenant);
Ok(Some(tenant_id))
}
@@ -420,7 +343,10 @@ pub fn get_tenant(tenant_id: TenantId, active_only: bool) -> anyhow::Result<Arc<
.get(&tenant_id)
.with_context(|| format!("Tenant {tenant_id} not found in the local state"))?;
if active_only && !tenant.is_active() {
anyhow::bail!("Tenant {tenant_id} is not active")
anyhow::bail!(
"Tenant {tenant_id} is not active. Current state: {:?}",
tenant.current_state()
)
} else {
Ok(Arc::clone(tenant))
}
@@ -497,374 +423,39 @@ pub async fn detach_tenant(
///
/// Get list of tenants, for the mgmt API
///
pub fn list_tenant_info(remote_index: &RemoteTimelineIndex) -> Vec<TenantInfo> {
pub fn list_tenants() -> Vec<(TenantId, TenantState)> {
tenants_state::read_tenants()
.iter()
.map(|(id, tenant)| {
let has_in_progress_downloads = remote_index
.tenant_entry(id)
.map(|entry| entry.has_in_progress_downloads());
// TODO this is not correct when we might have remote storage sync disabled:
// we keep `RemoteTimelineIndex` in memory anyway for simplicity and this error message is printed still
if has_in_progress_downloads.is_none() {
error!("timeline is not found in remote index while it is present in the tenants registry")
}
TenantInfo {
id: *id,
state: tenant.current_state(),
current_physical_size: None,
has_in_progress_downloads,
}
})
.map(|(id, tenant)| (*id, tenant.current_state()))
.collect()
}
#[derive(Debug)]
pub enum TenantAttachData {
Ready(HashMap<TimelineId, TimelineLocalFiles>),
Broken(anyhow::Error),
}
/// Attempts to collect information about all tenant and timelines, existing on the local FS.
/// If finds any, deletes all temporary files and directories, created before. Also removes empty directories,
/// that may appear due to such removals.
/// Does not fail on particular timeline or tenant collection errors, rather logging them and ignoring the entities.
fn local_tenant_timeline_files(
config: &'static PageServerConf,
) -> anyhow::Result<HashMap<TenantId, TenantAttachData>> {
let _entered = info_span!("local_tenant_timeline_files").entered();
let mut local_tenant_timeline_files = HashMap::new();
let tenants_dir = config.tenants_path();
for tenants_dir_entry in fs::read_dir(&tenants_dir)
.with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
{
match &tenants_dir_entry {
Ok(tenants_dir_entry) => {
let tenant_dir_path = tenants_dir_entry.path();
if is_temporary(&tenant_dir_path) {
info!(
"Found temporary tenant directory, removing: {}",
tenant_dir_path.display()
);
if let Err(e) = fs::remove_dir_all(&tenant_dir_path) {
error!(
"Failed to remove temporary directory '{}': {:?}",
tenant_dir_path.display(),
e
);
}
} else {
match collect_timelines_for_tenant(config, &tenant_dir_path) {
Ok((tenant_id, TenantAttachData::Broken(e))) => {
local_tenant_timeline_files.entry(tenant_id).or_insert(TenantAttachData::Broken(e));
},
Ok((tenant_id, TenantAttachData::Ready(collected_files))) => {
if collected_files.is_empty() {
match remove_if_empty(&tenant_dir_path) {
Ok(true) => info!("Removed empty tenant directory {}", tenant_dir_path.display()),
Ok(false) => {
// insert empty timeline entry: it has some non-temporary files inside that we cannot remove
// so make obvious for HTTP API callers, that something exists there and try to load the tenant
let _ = local_tenant_timeline_files.entry(tenant_id).or_insert_with(|| TenantAttachData::Ready(HashMap::new()));
},
Err(e) => error!("Failed to remove empty tenant directory: {e:?}"),
}
} else {
match local_tenant_timeline_files.entry(tenant_id) {
hash_map::Entry::Vacant(entry) => {
entry.insert(TenantAttachData::Ready(collected_files));
}
hash_map::Entry::Occupied(entry) =>{
if let TenantAttachData::Ready(old_timelines) = entry.into_mut() {
old_timelines.extend(collected_files);
}
},
}
}
},
Err(e) => error!(
"Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
tenants_dir.display(),
tenants_dir_entry,
e
),
}
/// Execute Attach mgmt API command.
///
/// Downloading all the tenant data is performed in the background, this merely
/// spawns the background task and returns quickly.
pub async fn attach_tenant(
conf: &'static PageServerConf,
tenant_id: TenantId,
remote_storage: &GenericRemoteStorage,
) -> anyhow::Result<()> {
match tenants_state::write_tenants().entry(tenant_id) {
hash_map::Entry::Occupied(e) => {
// Cannot attach a tenant that already exists. The error message depends on
// the state it's in.
match e.get().current_state() {
TenantState::Attaching => {
anyhow::bail!("tenant {tenant_id} attach is already in progress")
}
current_state => {
anyhow::bail!("tenant already exists, current state: {current_state:?}")
}
}
Err(e) => error!(
"Failed to list tenants dir entry {:?} in directory {}, reason: {:?}",
tenants_dir_entry,
tenants_dir.display(),
e
),
}
hash_map::Entry::Vacant(v) => {
let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage)?;
v.insert(tenant);
Ok(())
}
}
info!(
"Collected files for {} tenants",
local_tenant_timeline_files.len(),
);
Ok(local_tenant_timeline_files)
}
fn remove_if_empty(tenant_dir_path: &Path) -> anyhow::Result<bool> {
let directory_is_empty = tenant_dir_path
.read_dir()
.with_context(|| {
format!(
"Failed to read directory '{}' contents",
tenant_dir_path.display()
)
})?
.next()
.is_none();
if directory_is_empty {
fs::remove_dir_all(&tenant_dir_path).with_context(|| {
format!(
"Failed to remove empty directory '{}'",
tenant_dir_path.display(),
)
})?;
Ok(true)
} else {
Ok(false)
}
}
fn is_temporary(path: &Path) -> bool {
match path.file_name() {
Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
None => false,
}
}
fn is_uninit_mark(path: &Path) -> bool {
match path.file_name() {
Some(name) => name
.to_string_lossy()
.ends_with(TIMELINE_UNINIT_MARK_SUFFIX),
None => false,
}
}
fn collect_timelines_for_tenant(
config: &'static PageServerConf,
tenant_path: &Path,
) -> anyhow::Result<(TenantId, TenantAttachData)> {
let tenant_id = tenant_path
.file_name()
.and_then(OsStr::to_str)
.unwrap_or_default()
.parse::<TenantId>()
.context("Could not parse tenant id out of the tenant dir name")?;
let timelines_dir = config.timelines_path(&tenant_id);
if !timelines_dir.as_path().is_dir() {
return Ok((
tenant_id,
TenantAttachData::Broken(anyhow::anyhow!(
"Tenant {} has no timelines directory at {}",
tenant_id,
timelines_dir.display()
)),
));
}
let mut tenant_timelines = HashMap::new();
for timelines_dir_entry in fs::read_dir(&timelines_dir)
.with_context(|| format!("Failed to list timelines dir entry for tenant {tenant_id}"))?
{
match timelines_dir_entry {
Ok(timelines_dir_entry) => {
let timeline_dir = timelines_dir_entry.path();
if is_temporary(&timeline_dir) {
info!(
"Found temporary timeline directory, removing: {}",
timeline_dir.display()
);
if let Err(e) = fs::remove_dir_all(&timeline_dir) {
error!(
"Failed to remove temporary directory '{}': {:?}",
timeline_dir.display(),
e
);
}
} else if is_uninit_mark(&timeline_dir) {
let timeline_uninit_mark_file = &timeline_dir;
info!(
"Found an uninit mark file {}, removing the timeline and its uninit mark",
timeline_uninit_mark_file.display()
);
let timeline_id = timeline_uninit_mark_file
.file_stem()
.and_then(OsStr::to_str)
.unwrap_or_default()
.parse::<TimelineId>()
.with_context(|| {
format!(
"Could not parse timeline id out of the timeline uninit mark name {}",
timeline_uninit_mark_file.display()
)
})?;
let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
if let Err(e) =
remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file)
{
error!("Failed to clean up uninit marked timeline: {e:?}");
}
} else {
let timeline_id = timeline_dir
.file_name()
.and_then(OsStr::to_str)
.unwrap_or_default()
.parse::<TimelineId>()
.with_context(|| {
format!(
"Could not parse timeline id out of the timeline dir name {}",
timeline_dir.display()
)
})?;
let timeline_uninit_mark_file =
config.timeline_uninit_mark_file_path(tenant_id, timeline_id);
if timeline_uninit_mark_file.exists() {
info!("Found an uninit mark file for timeline {tenant_id}/{timeline_id}, removing the timeline and its uninit mark");
if let Err(e) = remove_timeline_and_uninit_mark(
&timeline_dir,
&timeline_uninit_mark_file,
) {
error!("Failed to clean up uninit marked timeline: {e:?}");
}
} else {
match collect_timeline_files(&timeline_dir) {
Ok((metadata, timeline_files)) => {
tenant_timelines.insert(
timeline_id,
TimelineLocalFiles::collected(metadata, timeline_files),
);
}
Err(e) => {
error!(
"Failed to process timeline dir contents at '{}', reason: {:?}",
timeline_dir.display(),
e
);
match remove_if_empty(&timeline_dir) {
Ok(true) => info!(
"Removed empty timeline directory {}",
timeline_dir.display()
),
Ok(false) => (),
Err(e) => {
error!("Failed to remove empty timeline directory: {e:?}")
}
}
}
}
}
}
}
Err(e) => {
error!("Failed to list timelines for entry tenant {tenant_id}, reason: {e:?}")
}
}
}
if tenant_timelines.is_empty() {
// this is normal, we've removed all broken, empty and temporary timeline dirs
// but should allow the tenant to stay functional and allow creating new timelines
// on a restart, we require tenants to have the timelines dir, so leave it on disk
debug!("Tenant {tenant_id} has no timelines loaded");
}
Ok((tenant_id, TenantAttachData::Ready(tenant_timelines)))
}
fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> {
fs::remove_dir_all(&timeline_dir)
.or_else(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
// we can leave the uninit mark without a timeline dir,
// just remove the mark then
Ok(())
} else {
Err(e)
}
})
.with_context(|| {
format!(
"Failed to remove unit marked timeline directory {}",
timeline_dir.display()
)
})?;
fs::remove_file(&uninit_mark).with_context(|| {
format!(
"Failed to remove timeline uninit mark file {}",
uninit_mark.display()
)
})?;
Ok(())
}
// discover timeline files and extract timeline metadata
// NOTE: ephemeral files are excluded from the list
fn collect_timeline_files(
timeline_dir: &Path,
) -> anyhow::Result<(TimelineMetadata, HashMap<PathBuf, LayerFileMetadata>)> {
let mut timeline_files = HashMap::new();
let mut timeline_metadata_path = None;
let timeline_dir_entries =
fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?;
for entry in timeline_dir_entries {
let entry_path = entry.context("Failed to list timeline dir entry")?.path();
let metadata = entry_path.metadata()?;
if metadata.is_file() {
if entry_path.file_name().and_then(OsStr::to_str) == Some(METADATA_FILE_NAME) {
timeline_metadata_path = Some(entry_path);
} else if is_ephemeral_file(&entry_path.file_name().unwrap().to_string_lossy()) {
debug!("skipping ephemeral file {}", entry_path.display());
continue;
} else if is_temporary(&entry_path) {
info!("removing temp timeline file at {}", entry_path.display());
fs::remove_file(&entry_path).with_context(|| {
format!(
"failed to remove temp download file at {}",
entry_path.display()
)
})?;
} else {
let layer_metadata = LayerFileMetadata::new(metadata.len());
timeline_files.insert(entry_path, layer_metadata);
}
}
}
// FIXME (rodionov) if attach call succeeded, and then pageserver is restarted before download is completed
// then attach is lost. There would be no retries for that,
// initial collect will fail because there is no metadata.
// We either need to start download if we see empty dir after restart or attach caller should
// be aware of that and retry attach if awaits_download for timeline switched from true to false
// but timelinne didn't appear locally.
// Check what happens with remote index in that case.
let timeline_metadata_path = match timeline_metadata_path {
Some(path) => path,
None => anyhow::bail!("No metadata file found in the timeline directory"),
};
let metadata = TimelineMetadata::from_bytes(
&fs::read(&timeline_metadata_path).context("Failed to read timeline metadata file")?,
)
.context("Failed to parse timeline metadata file bytes")?;
anyhow::ensure!(
metadata.ancestor_timeline().is_some() || !timeline_files.is_empty(),
"Timeline has no ancestor and no layer files"
);
Ok((metadata, timeline_files))
}
pageserver/src/walingest.rs
+12 -12
View File
@@ -1090,9 +1090,9 @@ mod tests {
Ok(walingest)
}
#[test]
fn test_relsize() -> Result<()> {
let tenant = TenantHarness::create("test_relsize")?.load();
#[tokio::test]
async fn test_relsize() -> Result<()> {
let tenant = TenantHarness::create("test_relsize")?.load().await;
let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
let mut walingest = init_walingest_test(&*tline)?;
@@ -1218,9 +1218,9 @@ mod tests {
// Test what happens if we dropped a relation
// and then created it again within the same layer.
#[test]
fn test_drop_extend() -> Result<()> {
let tenant = TenantHarness::create("test_drop_extend")?.load();
#[tokio::test]
async fn test_drop_extend() -> Result<()> {
let tenant = TenantHarness::create("test_drop_extend")?.load().await;
let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
let mut walingest = init_walingest_test(&*tline)?;
@@ -1258,9 +1258,9 @@ mod tests {
// Test what happens if we truncated a relation
// so that one of its segments was dropped
// and then extended it again within the same layer.
#[test]
fn test_truncate_extend() -> Result<()> {
let tenant = TenantHarness::create("test_truncate_extend")?.load();
#[tokio::test]
async fn test_truncate_extend() -> Result<()> {
let tenant = TenantHarness::create("test_truncate_extend")?.load().await;
let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
let mut walingest = init_walingest_test(&*tline)?;
@@ -1346,9 +1346,9 @@ mod tests {
/// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
/// split into multiple 1 GB segments in Postgres.
#[test]
fn test_large_rel() -> Result<()> {
let tenant = TenantHarness::create("test_large_rel")?.load();
#[tokio::test]
async fn test_large_rel() -> Result<()> {
let tenant = TenantHarness::create("test_large_rel")?.load().await;
let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
let mut walingest = init_walingest_test(&*tline)?;
pageserver/src/walreceiver/connection_manager.rs
+13 -12
View File
@@ -857,10 +857,10 @@ mod tests {
use super::*;
use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
#[test]
fn no_connection_no_candidate() -> anyhow::Result<()> {
#[tokio::test]
async fn no_connection_no_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("no_connection_no_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let now = Utc::now().naive_utc();
let lagging_wal_timeout = chrono::Duration::from_std(state.lagging_wal_timeout)?;
@@ -951,7 +951,7 @@ mod tests {
#[tokio::test]
async fn connection_no_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("connection_no_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let now = Utc::now().naive_utc();
let connected_sk_id = NodeId(0);
@@ -1045,10 +1045,10 @@ mod tests {
Ok(())
}
#[test]
fn no_connection_candidate() -> anyhow::Result<()> {
#[tokio::test]
async fn no_connection_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("no_connection_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let now = Utc::now().naive_utc();
state.wal_connection = None;
@@ -1161,7 +1161,7 @@ mod tests {
#[tokio::test]
async fn candidate_with_many_connection_failures() -> anyhow::Result<()> {
let harness = TenantHarness::create("candidate_with_many_connection_failures")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let now = Utc::now().naive_utc();
let current_lsn = Lsn(100_000).align();
@@ -1229,7 +1229,7 @@ mod tests {
#[tokio::test]
async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let current_lsn = Lsn(100_000).align();
let now = Utc::now().naive_utc();
@@ -1320,7 +1320,7 @@ mod tests {
#[tokio::test]
async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let current_lsn = Lsn(100_000).align();
let now = Utc::now().naive_utc();
@@ -1394,7 +1394,7 @@ mod tests {
#[tokio::test]
async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?;
let mut state = dummy_state(&harness);
let mut state = dummy_state(&harness).await;
let current_lsn = Lsn(100_000).align();
let new_lsn = Lsn(100_100).align();
let now = Utc::now().naive_utc();
@@ -1472,7 +1472,7 @@ mod tests {
const DUMMY_SAFEKEEPER_CONNSTR: &str = "safekeeper_connstr";
fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
WalreceiverState {
id: TenantTimelineId {
tenant_id: harness.tenant_id,
@@ -1480,6 +1480,7 @@ mod tests {
},
timeline: harness
.load()
.await
.create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION)
.expect("Failed to create an empty timeline for dummy wal connection manager")
.initialize()
pageserver/src/walreceiver/walreceiver_connection.rs
+3 -19
View File
@@ -26,13 +26,12 @@ use crate::{
task_mgr::TaskKind,
task_mgr::WALRECEIVER_RUNTIME,
tenant::{Timeline, WalReceiverInfo},
tenant_mgr,
walingest::WalIngest,
walrecord::DecodedWALRecord,
};
use postgres_ffi::waldecoder::WalStreamDecoder;
use pq_proto::ReplicationFeedback;
use utils::{id::TenantTimelineId, lsn::Lsn};
use utils::lsn::Lsn;
/// Status of the connection.
#[derive(Debug, Clone)]
@@ -139,10 +138,6 @@ pub async fn handle_walreceiver_connection(
return Ok(());
}
let tenant_id = timeline.tenant_id;
let timeline_id = timeline.timeline_id;
let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
//
// Start streaming the WAL, from where we left off previously.
//
@@ -291,19 +286,8 @@ pub async fn handle_walreceiver_connection(
})?;
if let Some(last_lsn) = status_update {
let remote_index = tenant.get_remote_index();
let timeline_remote_consistent_lsn = remote_index
.read()
.await
// here we either do not have this timeline in remote index
// or there were no checkpoints for it yet
.timeline_entry(&TenantTimelineId {
tenant_id,
timeline_id,
})
.map(|remote_timeline| remote_timeline.metadata.disk_consistent_lsn())
// no checkpoint was uploaded
.unwrap_or(Lsn(0));
let timeline_remote_consistent_lsn =
timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
// The last LSN we processed. It is not guaranteed to survive pageserver crash.
let write_lsn = u64::from(last_lsn);
test_runner/regress/test_broken_timeline.py
+4 -2
View File
@@ -77,7 +77,9 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
# But all others are broken
# First timeline would not get loaded into pageserver due to corrupt metadata file
with pytest.raises(Exception, match=f"Timeline {tenant1}/{timeline1} was not found") as err:
with pytest.raises(
Exception, match=f"Tenant {tenant1} is not active. Current state: Broken"
) as err:
pg1.start()
log.info(
f"As expected, compute startup failed eagerly for timeline with corrupt metadata: {err}"
@@ -86,7 +88,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
# Second timeline has no ancestors, only the metadata file and no layer files
# We don't have the remote storage enabled, which means timeline is in an incorrect state,
# it's not loaded at all
with pytest.raises(Exception, match=f"Timeline {tenant2}/{timeline2} was not found") as err:
with pytest.raises(Exception, match=".*extracting base backup failed") as err:
pg2.start()
log.info(f"As expected, compute startup failed for timeline with missing layers: {err}")