diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index af9be2d456..2c42e5672a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -15,8 +15,14 @@ use bytes::{BufMut, Bytes, BytesMut}; /// A state of a tenant in pageserver's memory. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum TenantState { + // This tenant is being loaded from local disk + Loading, + // This tenant is being downloaded from cloud storage. + Attaching, /// Tenant is fully operational, its background jobs might be running or not. - Active { background_jobs_running: bool }, + Active { + background_jobs_running: bool, + }, /// A tenant is recognized by pageserver, but not yet ready to operate: /// e.g. not present locally and being downloaded or being read into memory from the file system. Paused, @@ -144,7 +150,6 @@ pub struct TenantInfo { pub id: TenantId, pub state: TenantState, pub current_physical_size: Option, // physical size is only included in `tenant_status` endpoint - pub has_in_progress_downloads: Option, } /// This represents the output of the "timeline_detail" and "timeline_list" API calls. @@ -168,6 +173,8 @@ pub struct TimelineInfo { pub latest_gc_cutoff_lsn: Lsn, #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, + #[serde_as(as = "DisplayFromStr")] + pub remote_consistent_lsn: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded pub current_physical_size: Option, // is None when timeline is Unloaded pub current_logical_size_non_incremental: Option, @@ -180,10 +187,6 @@ pub struct TimelineInfo { pub last_received_msg_ts: Option, pub pg_version: u32, - #[serde_as(as = "Option")] - pub remote_consistent_lsn: Option, - pub awaits_download: bool, - pub state: TimelineState, // Some of the above fields are duplicated in 'local' and 'remote', for backwards- diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 62119b51c6..9434142848 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -264,7 +264,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { }) .transpose() .context("Failed to init generic remote storage")?; - let remote_index = { + { let _rt_guard = BACKGROUND_RUNTIME.enter(); tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())? }; @@ -276,7 +276,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { { let _rt_guard = MGMT_REQUEST_RUNTIME.enter(); - let router = http::make_router(conf, auth.clone(), remote_index, remote_storage)?; + let router = http::make_router(conf, auth.clone(), remote_storage)?; let service = utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap(); let server = hyper::Server::from_tcp(http_listener)? diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index db581efc7d..b9d1288ea3 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -7,15 +7,12 @@ use remote_storage::GenericRemoteStorage; use tokio::task::JoinError; use tracing::*; -use super::models::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo}; use super::models::{ - StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo, - TimelineCreateRequest, + LocalTimelineInfo, RemoteTimelineInfo, StatusResponse, TenantConfigRequest, + TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo, }; use crate::pgdatadir_mapping::LsnForTimestamp; -use crate::storage_sync; -use crate::storage_sync::index::{RemoteIndex, RemoteTimeline}; -use crate::tenant::{TenantState, Timeline}; +use crate::tenant::Timeline; use crate::tenant_config::TenantConfOpt; use crate::{config::PageServerConf, tenant_mgr}; use utils::{ @@ -27,7 +24,7 @@ use utils::{ request::parse_request_param, RequestExt, RouterBuilder, }, - id::{TenantId, TenantTimelineId, TimelineId}, + id::{TenantId, TimelineId}, lsn::Lsn, }; @@ -40,7 +37,6 @@ use crate::CheckpointConfig; struct State { conf: &'static PageServerConf, auth: Option>, - remote_index: RemoteIndex, allowlist_routes: Vec, remote_storage: Option, } @@ -49,7 +45,6 @@ impl State { fn new( conf: &'static PageServerConf, auth: Option>, - remote_index: RemoteIndex, remote_storage: Option, ) -> anyhow::Result { let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"] @@ -60,7 +55,6 @@ impl State { conf, auth, allowlist_routes, - remote_index, remote_storage, }) } @@ -80,12 +74,24 @@ fn get_config(request: &Request) -> &'static PageServerConf { } // Helper function to construct a TimelineInfo struct for a timeline -async fn build_timeline_info( - state: &State, +fn build_timeline_info( timeline: &Arc, include_non_incremental_logical_size: bool, include_non_incremental_physical_size: bool, ) -> anyhow::Result { + let mut info = build_timeline_info_common(timeline)?; + if include_non_incremental_logical_size { + info.current_logical_size_non_incremental = + Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?); + } + if include_non_incremental_physical_size { + info.current_physical_size_non_incremental = + Some(timeline.get_physical_size_non_incremental()?) + } + Ok(info) +} + +fn build_timeline_info_common(timeline: &Arc) -> anyhow::Result { let last_record_lsn = timeline.get_last_record_lsn(); let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = { let guard = timeline.last_received_wal.lock().unwrap(); @@ -100,22 +106,6 @@ async fn build_timeline_info( } }; - let (remote_consistent_lsn, awaits_download) = if let Some(remote_entry) = state - .remote_index - .read() - .await - .timeline_entry(&TenantTimelineId { - tenant_id: timeline.tenant_id, - timeline_id: timeline.timeline_id, - }) { - ( - Some(remote_entry.metadata.disk_consistent_lsn()), - remote_entry.awaits_download, - ) - } else { - (None, false) - }; - let ancestor_timeline_id = timeline.get_ancestor_timeline_id(); let ancestor_lsn = match timeline.get_ancestor_lsn() { Lsn(0) => None, @@ -130,6 +120,7 @@ async fn build_timeline_info( }; let current_physical_size = Some(timeline.get_physical_size()); let state = timeline.current_state(); + let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0)); let info = TimelineInfo { tenant_id: timeline.tenant_id, @@ -137,28 +128,19 @@ async fn build_timeline_info( ancestor_timeline_id, ancestor_lsn, disk_consistent_lsn: timeline.get_disk_consistent_lsn(), + remote_consistent_lsn, last_record_lsn, prev_record_lsn: Some(timeline.get_prev_record_lsn()), latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(), current_logical_size, current_physical_size, - current_logical_size_non_incremental: if include_non_incremental_logical_size { - Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?) - } else { - None - }, - current_physical_size_non_incremental: if include_non_incremental_physical_size { - Some(timeline.get_physical_size_non_incremental()?) - } else { - None - }, + current_logical_size_non_incremental: None, + current_physical_size_non_incremental: None, wal_source_connstr, last_received_msg_lsn, last_received_msg_ts, pg_version: timeline.pg_version, - remote_consistent_lsn, - awaits_download, state, // Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility @@ -170,7 +152,7 @@ async fn build_timeline_info( current_physical_size, }, remote: RemoteTimelineInfo { - remote_consistent_lsn, + remote_consistent_lsn: Some(remote_consistent_lsn), }, }; Ok(info) @@ -187,34 +169,28 @@ async fn timeline_create_handler(mut request: Request) -> Result { - // Created. Construct a TimelineInfo for it. - let timeline_info = build_timeline_info(state, &new_timeline, false, false) - .await - .map_err(ApiError::InternalServerError)?; - Ok(Some(timeline_info)) - } - Ok(None) => Ok(None), // timeline already exists - Err(err) => Err(ApiError::InternalServerError(err)), + match tenant.create_timeline( + new_timeline_id, + request_data.ancestor_timeline_id.map(TimelineId::from), + request_data.ancestor_start_lsn, + request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION) + ) + .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) + .await { + Ok(Some(new_timeline)) => { + // Created. Construct a TimelineInfo for it. + let timeline_info = build_timeline_info_common(&new_timeline) + .map_err(ApiError::InternalServerError)?; + json_response(StatusCode::CREATED, timeline_info) } + Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists + Err(err) => Err(ApiError::InternalServerError(err)), } - .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) - .await?; - - Ok(match new_timeline_info { - Some(info) => json_response(StatusCode::CREATED, info)?, - None => json_response(StatusCode::CONFLICT, ())?, - }) } async fn timeline_list_handler(request: Request) -> Result, ApiError> { @@ -225,22 +201,20 @@ async fn timeline_list_handler(request: Request) -> Result, query_param_present(&request, "include-non-incremental-physical-size"); check_permission(&request, Some(tenant_id))?; - let state = get_state(&request); + let _entered = info_span!("timeline_list", tenant = %tenant_id).entered(); - let timelines = info_span!("timeline_list", tenant = %tenant_id).in_scope(|| { + let timelines = { let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?; - Ok(tenant.list_timelines()) - })?; + tenant.list_timelines() + }; let mut response_data = Vec::with_capacity(timelines.len()); for timeline in timelines { let timeline_info = build_timeline_info( - state, &timeline, include_non_incremental_logical_size, include_non_incremental_physical_size, ) - .await .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}") .map_err(ApiError::InternalServerError)?; @@ -289,8 +263,6 @@ async fn timeline_detail_handler(request: Request) -> Result) -> Result) -> Result, info!("Handling tenant attach {tenant_id}"); - tokio::task::spawn_blocking(move || match tenant_mgr::get_tenant(tenant_id, false) { - Ok(tenant) => { - if tenant.list_timelines().is_empty() { - info!("Attaching to tenant {tenant_id} with zero timelines"); - Ok(()) - } else { - Err(ApiError::Conflict( - "Tenant is already present locally".to_owned(), - )) - } - } - Err(_) => Ok(()), - }) - .await - .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??; - let state = get_state(&request); - let remote_index = &state.remote_index; - let mut index_accessor = remote_index.write().await; - if let Some(tenant_entry) = index_accessor.tenant_entry_mut(&tenant_id) { - if tenant_entry.has_in_progress_downloads() { - return Err(ApiError::Conflict( - "Tenant download is already in progress".to_string(), - )); - } - - for (timeline_id, remote_timeline) in tenant_entry.iter_mut() { - storage_sync::schedule_layer_download(tenant_id, *timeline_id); - remote_timeline.awaits_download = true; - } - return json_response(StatusCode::ACCEPTED, ()); - } - // no tenant in the index, release the lock to make the potentially lengthy download operation - drop(index_accessor); - - // download index parts for every tenant timeline - let remote_timelines = match gather_tenant_timelines_index_parts(state, tenant_id).await { - Ok(Some(remote_timelines)) => remote_timelines, - Ok(None) => return Err(ApiError::NotFound(anyhow!("Unknown remote tenant"))), - Err(e) => { - error!("Failed to retrieve remote tenant data: {:?}", e); - return Err(ApiError::NotFound(anyhow!( - "Failed to retrieve remote tenant" - ))); - } - }; - - // recheck that download is not in progress because - // we've released the lock to avoid holding it during the download - let mut index_accessor = remote_index.write().await; - let tenant_entry = match index_accessor.tenant_entry_mut(&tenant_id) { - Some(tenant_entry) => { - if tenant_entry.has_in_progress_downloads() { - return Err(ApiError::Conflict( - "Tenant download is already in progress".to_string(), - )); - } - tenant_entry - } - None => index_accessor.add_tenant_entry(tenant_id), - }; - - // populate remote index with the data from index part and create directories on the local filesystem - for (timeline_id, mut remote_timeline) in remote_timelines { - tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id)) + if let Some(remote_storage) = &state.remote_storage { + // FIXME: distinguish between "Tenant already exists" and other errors + tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage) + .instrument(info_span!("tenant_attach", tenant = %tenant_id)) .await - .context("Failed to create new timeline directory") .map_err(ApiError::InternalServerError)?; - - remote_timeline.awaits_download = true; - tenant_entry.insert(timeline_id, remote_timeline); - // schedule actual download - storage_sync::schedule_layer_download(tenant_id, timeline_id); + } else { + return Err(ApiError::BadRequest(anyhow!( + "attach_tenant is possible because pageserver was configured without remote storage" + ))); } json_response(StatusCode::ACCEPTED, ()) } -/// Note: is expensive from s3 access perspective, -/// for details see comment to `storage_sync::gather_tenant_timelines_index_parts` -async fn gather_tenant_timelines_index_parts( - state: &State, - tenant_id: TenantId, -) -> anyhow::Result>> { - let index_parts = match state.remote_storage.as_ref() { - Some(storage) => { - storage_sync::gather_tenant_timelines_index_parts(state.conf, storage, tenant_id).await - } - None => return Ok(None), - } - .with_context(|| format!("Failed to download index parts for tenant {tenant_id}"))?; - - let mut remote_timelines = Vec::with_capacity(index_parts.len()); - for (timeline_id, index_part) in index_parts { - let timeline_path = state.conf.timeline_path(&timeline_id, &tenant_id); - let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part) - .with_context(|| { - format!("Failed to convert index part into remote timeline for timeline {tenant_id}/{timeline_id}") - })?; - remote_timelines.push((timeline_id, remote_timeline)); - } - Ok(Some(remote_timelines)) -} - async fn timeline_delete_handler(request: Request) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; - let state = get_state(&request); tenant_mgr::delete_timeline(tenant_id, timeline_id) .instrument(info_span!("timeline_delete", tenant = %tenant_id, timeline = %timeline_id)) .await @@ -470,12 +351,6 @@ async fn timeline_delete_handler(request: Request) -> Result) -> Result, // Replace this with better handling once the error type permits it. .map_err(ApiError::InternalServerError)?; - let mut remote_index = state.remote_index.write().await; - remote_index.remove_tenant_entry(&tenant_id); - json_response(StatusCode::OK, ()) } async fn tenant_list_handler(request: Request) -> Result, ApiError> { check_permission(&request, None)?; - let state = get_state(&request); - // clone to avoid holding the lock while awaiting for blocking task - let remote_index = state.remote_index.read().await.clone(); - let response_data = tokio::task::spawn_blocking(move || { let _enter = info_span!("tenant_list").entered(); - crate::tenant_mgr::list_tenant_info(&remote_index) + tenant_mgr::list_tenants() + .iter() + .map(|(id, state)| TenantInfo { + id: *id, + state: *state, + current_physical_size: None, + }) + .collect::>() }) .await .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?; @@ -519,58 +394,36 @@ async fn tenant_status(request: Request) -> Result, ApiErro let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - // if tenant is in progress of downloading it can be absent in global tenant map - let tenant = tenant_mgr::get_tenant(tenant_id, false); + let tenant_info = tokio::task::spawn_blocking(move || { + let _enter = info_span!("tenant_status_handler", tenant = %tenant_id).entered(); + let tenant = tenant_mgr::get_tenant(tenant_id, false)?; - let state = get_state(&request); - let remote_index = &state.remote_index; - - let index_accessor = remote_index.read().await; - let has_in_progress_downloads = index_accessor - .tenant_entry(&tenant_id) - .map(|t| t.has_in_progress_downloads()) - .unwrap_or_else(|| { - info!("Tenant {tenant_id} not found in remote index"); - false - }); - - let (tenant_state, current_physical_size) = match tenant { - Ok(tenant) => { - let timelines = tenant.list_timelines(); - // Calculate total physical size of all timelines - let mut current_physical_size = 0; - for timeline in timelines { - current_physical_size += timeline.get_physical_size(); - } - - (tenant.current_state(), Some(current_physical_size)) + // Calculate total physical size of all timelines + let mut current_physical_size = 0; + for timeline in tenant.list_timelines().iter() { + current_physical_size += timeline.get_physical_size(); } - Err(e) => { - error!("Failed to get local tenant state: {e:#}"); - if has_in_progress_downloads { - (TenantState::Paused, None) - } else { - (TenantState::Broken, None) - } - } - }; - json_response( - StatusCode::OK, - TenantInfo { + let tenant_info = TenantInfo { id: tenant_id, - state: tenant_state, - current_physical_size, - has_in_progress_downloads: Some(has_in_progress_downloads), - }, - ) + state: tenant.current_state(), + current_physical_size: Some(current_physical_size), + }; + + Ok::<_, anyhow::Error>(tenant_info) + }) + .await + .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))? + .map_err(ApiError::InternalServerError)?; + + json_response(StatusCode::OK, tenant_info) } async fn tenant_size_handler(request: Request) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::InternalServerError)?; + let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::InternalServerError)?; // this can be long operation, it currently is not backed by any request coalescing or similar let inputs = tenant @@ -618,8 +471,6 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result) -> Result, ApiError> { pub fn make_router( conf: &'static PageServerConf, auth: Option>, - remote_index: RemoteIndex, remote_storage: Option, ) -> anyhow::Result> { let spec = include_bytes!("openapi_spec.yml"); @@ -928,8 +783,7 @@ pub fn make_router( Ok(router .data(Arc::new( - State::new(conf, auth, remote_index, remote_storage) - .context("Failed to initialize router state")?, + State::new(conf, auth, remote_storage).context("Failed to initialize router state")?, )) .get("/v1/status", status_handler) .put( diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index b62143f650..49725deff6 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -22,10 +22,7 @@ pub mod walreceiver; pub mod walrecord; pub mod walredo; -use std::collections::HashMap; - use tracing::info; -use utils::id::{TenantId, TimelineId}; use crate::task_mgr::TaskKind; @@ -71,7 +68,7 @@ pub async fn shutdown_pageserver(exit_code: i32) { // // FIXME: Does this wait for the sync tasks to finish syncing what's queued up? // Should it? - task_mgr::shutdown_tasks(Some(TaskKind::StorageSync), None, None).await; + task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await; // Shut down the HTTP endpoint last, so that you can still check the server's // status while it's shutting down. @@ -106,24 +103,17 @@ fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds } } -/// A newtype to store arbitrary data grouped by tenant and timeline ids. -/// One could use [`utils::id::TenantTimelineId`] for grouping, but that would -/// not include the cases where a certain tenant has zero timelines. -/// This is sometimes important: a tenant could be registered during initial load from FS, -/// even if he has no timelines on disk. -#[derive(Debug)] -pub struct TenantTimelineValues(HashMap>); - -impl TenantTimelineValues { - fn new() -> Self { - Self(HashMap::new()) - } -} - /// A suffix to be used during file sync from the remote storage, /// to ensure that we do not leave corrupted files that pretend to be layers. const TEMP_FILE_SUFFIX: &str = "___temp"; +pub fn is_temporary(path: &std::path::Path) -> bool { + match path.file_name() { + Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX), + None => false, + } +} + #[cfg(test)] mod backoff_defaults_tests { use super::*; diff --git a/pageserver/src/storage_sync.rs b/pageserver/src/storage_sync.rs index 037fe76d7f..428c39cff3 100644 --- a/pageserver/src/storage_sync.rs +++ b/pageserver/src/storage_sync.rs @@ -1,79 +1,73 @@ -//! There are a few components the storage machinery consists of: +//! This module manages synchronizing local fs with remote storage. It consists of: //! -//! * [`RemoteStorage`] that is used to interact with an arbitrary external storage +//! * [`RemoteTimelineClient`] provides functions related to upload/download of a particular timeline. +//! It contains a queue of pending uploads, and manages the queue, performing uploads in parallel +//! when it's safe to do so. //! -//! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync. -//! Synchronization internals are split into submodules -//! * [`storage_sync::index`] to keep track of remote tenant files, the metadata and their mappings to local files -//! * [`storage_sync::upload`] and [`storage_sync::download`] to manage archive creation and upload; download and extraction, respectively +//! * Stand-alone function, [`list_remote_timelines`], to get list of timelines of a tenant. //! -//! * public API via to interact with the external world: -//! * [`start_local_timeline_sync`] to launch a background async loop to handle the synchronization -//! * [`schedule_layer_upload`], [`schedule_layer_download`], and[`schedule_layer_delete`] to enqueue a new task -//! to be processed by the async loop +//! These functions use the low-level remote storage client, [`RemoteStorage`], which is defined +//! in the remote_storage crate (see libs/remote_storage). //! -//! Here's a schematic overview of all interactions backup and the rest of the pageserver perform: + +//! FIXME: Update below as needed //! -//! +------------------------+ +--------->-------+ -//! | | - - - (init async loop) - - - -> | | -//! | | | | -//! | | -------------------------------> | async | -//! | pageserver | (enqueue timeline sync task) | upload/download | -//! | | | loop | -//! | | <------------------------------- | | -//! | | (apply new timeline sync states) | | -//! +------------------------+ +---------<-------+ -//! | -//! | -//! CRUD layer file operations | -//! (upload/download/delete/list, etc.) | -//! V -//! +------------------------+ -//! | | -//! | [`RemoteStorage`] impl | -//! | | -//! | pageserver assumes it | -//! | owns exclusive write | -//! | access to this storage | -//! +------------------------+ +//! At system startup, the pageserver will load all tenants that are found on +//! local disk. A RemoteTimelineClient is initialized for each timeline. The +//! upload queue of each RemoteTimelineClient is initially empty, but if there +//! are any files on local filesystem that don't exist remotely, they are added +//! to the upload queue at startup. //! -//! First, during startup, the pageserver inits the storage sync task with the async loop, or leaves the loop uninitialised, if configured so. -//! The loop inits the storage connection and checks the remote files stored. -//! This is done once at startup only, relying on the fact that pageserver uses the storage alone (ergo, nobody else uploads the files to the storage but this server). -//! Based on the remote storage data, the sync logic immediately schedules sync tasks for local timelines and reports about remote only timelines to pageserver, so it can -//! query their downloads later if they are accessed. +//! Some time later, during pageserver checkpoints, in-memory data is flushed to +//! disk along with its metadata. Whenever a new layer file is created, the +//! pageserver also schedules it for upload, by calling schedule_layer_upload. +//! At a checkpoint, after all the new layer files have been created, the +//! updated metadata file is also scheduled for upload, by calling +//! schedule_index_upload. See [`crate::tenant::timeline`] for the upload calls +//! and the adjacent logic. //! -//! Some time later, during pageserver checkpoints, in-memory data is flushed onto disk along with its metadata. -//! If the storage sync loop was successfully started before, pageserver schedules the layer files and the updated metadata file for upload, every time a layer is flushed to disk. -//! The uploads are disabled, if no remote storage configuration is provided (no sync loop is started this way either). -//! See [`crate::tenant`] for the upload calls and the adjacent logic. +//! If no remote storage configuration is provided, the RemoetTimelineClient is +//! not created and the uploads are skipped. //! -//! Synchronization logic is able to communicate back with updated timeline sync states, submitted via [`crate::tenant_mgr::attach_local_tenants`] function. -//! Tenant manager applies corresponding timeline updates in pageserver's in-memory state. -//! Such submissions happen in two cases: -//! * once after the sync loop startup, to signal pageserver which timelines will be synchronized in the near future -//! * after every loop step, in case a timeline needs to be reloaded or evicted from pageserver's memory +//! If a timeline is deleted, its RemoteTimelineClient is dropped, cancelling +//! all pending uploads. If a timeline is detached, TODO what happens ??? //! -//! When the pageserver terminates, the sync loop finishes current sync task (if any) and exits. +//! If the pageserver terminates, TODO what happens ??? //! -//! The storage logic considers `image` as a set of local files (layers), fully representing a certain timeline at given moment (identified with `disk_consistent_lsn` from the corresponding `metadata` file). -//! Timeline can change its state, by adding more files on disk and advancing its `disk_consistent_lsn`: this happens after pageserver checkpointing and is followed -//! by the storage upload, if enabled. -//! Yet timeline cannot alter already existing files, and cannot remove those too: only a GC process is capable of removing unused files. -//! This way, remote storage synchronization relies on the fact that every checkpoint is incremental and local files are "immutable": -//! * when a certain checkpoint gets uploaded, the sync loop remembers the fact, preventing further reuploads of the same state -//! * no files are deleted from either local or remote storage, only the missing ones locally/remotely get downloaded/uploaded, local metadata file will be overwritten -//! when the newer image is downloaded +//! To have a consistent set of files, it's important that uploads and deletions +//! are performed in the right order. For example, the index file contains a +//! list of layer files, so it must not be uploaded until all the layer files +//! that are in its list have been succesfully uploaded. RemoteTimelineClient +//! maintains a queue of operations, and it knows which operations can be +//! performed in parallel, and which operations act like a "barrier" that +//! require preceding operations to finish. The calling code just needs to call +//! the schedule-functions in the correct order, and RemoteTimelineClient will +//! parallelize the operations in a way that's safe. //! -//! Pageserver maintains similar to the local file structure remotely: all layer files are uploaded with the same names under the same directory structure. +//! The caller should be careful with deletion, though. You should not delete +//! files locally that haven't been uploaded yet. Otherwise the upload will +//! fail. After scheduling uploads, you can use the 'wait_completion' function +//! to wait for them to finish. +//! +//! - We rely on read-after write consistency in the remote storage. +//! - Layer files are immutable + +//! +//! The "directory structure" in the remote storage mirrors the local directory structure, with paths +//! like `tenants//timelines//. //! Yet instead of keeping the `metadata` file remotely, we wrap it with more data in [`IndexPart`], containing the list of remote files. //! This file gets read to populate the cache, if the remote timeline data is missing from it and gets updated after every successful download. //! This way, we optimize S3 storage access by not running the `S3 list` command that could be expencive and slow: knowing both [`TenantId`] and [`TimelineId`], //! we can always reconstruct the path to the timeline, use this to get the same path on the remote storage and retrieve its shard contents, if needed, same as any layer files. +//! TODO: update this paragraph with more details on how the list of remote files is kept up-to-date //! -//! By default, pageserver reads the remote storage index data only for timelines located locally, to synchronize those, if needed. -//! Bulk index data download happens only initially, on pageserver startup. The rest of the remote storage stays unknown to pageserver and loaded on demand only, -//! when a new timeline is scheduled for the download. +//! At pageserver startup, it downloads the index files of every locally-present +//! tenant, to synchronize the local state with remote storage. Other tenants +//! that might be present in the remote storage are ignored. When a tenant is +//! attached to the pageserver, the directory structure for the tenant and all +//! its timelines are created on local disk, and the timeline index files are +//! downloaded, to create the local metadata files. The layer files containing +//! the actual data are downloaded on-demand. //! //! NOTES: //! * pageserver assumes it has exclusive write access to the remote storage. If supported, the way multiple pageservers can be separated in the same storage @@ -82,22 +76,14 @@ //! * the sync tasks may not processed immediately after the submission: if they error and get re-enqueued, their execution might be backed off to ensure error cap is not exceeded too fast. //! The sync queue processing also happens in batches, so the sync tasks can wait in the queue for some time. //! -//! A synchronization logic for the [`RemoteStorage`] and pageserver in-memory state to ensure correct synchronizations -//! between local tenant files and their counterparts from the remote storage. +//! Uploads are queued and executed in the background and in parallel, enforcing the ordering rules. +//! Downloads are performed immediately, and independently of the uploads. //! -//! The synchronization does not aim to be immediate, yet eventually consistent. -//! Synchronization is done with the queue being emptied via separate thread asynchronously, -//! attempting to fully store pageserver's local data on the remote storage in a custom format, beneficial for storing. -//! -//! A queue is implemented in the [`sync_queue`] module as a VecDeque to hold the tasks, and a condition variable for blocking when the queue is empty. -//! -//! The queue gets emptied by a single thread with the loop, that polls the tasks in batches of deduplicated tasks. -//! A task from the batch corresponds to a single timeline, with its files to sync merged together: given that only one task sync loop step is active at a time, -//! timeline uploads and downloads can happen concurrently, in no particular order due to incremental nature of the timeline layers. //! Deletion happens only after a successful upload only, otherwise the compaction output might make the timeline inconsistent until both tasks are fully processed without errors. //! Upload and download update the remote data (inmemory index and S3 json index part file) only after every layer is successfully synchronized, while the deletion task //! does otherwise: it requires to have the remote data updated first successfully: blob files will be invisible to pageserver this way. //! +//! FIXME: how is the initial list of remote files created now? Update this paragraph //! During the loop startup, an initial [`RemoteTimelineIndex`] state is constructed via downloading and merging the index data for all timelines, //! present locally. //! It's enough to poll such timelines' remote state once on startup only, due to an agreement that only one pageserver at a time has an exclusive @@ -117,10 +103,9 @@ //! If the index part does not contain some file path but it's present remotely, such file is invisible to pageserver and ignored. //! Among other tasks, the index is used to prevent invalid uploads and non-existing downloads on demand, refer to [`index`] for more details. //! +//! FIXME: update this paragraph //! Index construction is currently the only place where the storage sync can return an [`Err`] to the user. -//! New sync tasks are accepted via [`schedule_layer_upload`], [`schedule_layer_download`] and [`schedule_layer_delete`] functions, -//! disregarding of the corresponding loop startup. -//! It's up to the caller to avoid synchronizations if the loop is disabled: otherwise, the sync tasks will be ignored. +//! New sync tasks are accepted via [`schedule_layer_upload`], [`schedule_layer_download`] and [`schedule_layer_delete`] functions. //! After the initial state is loaded into memory and the loop starts, any further [`Err`] results do not stop the loop, but rather //! reschedule the same task, with possibly less files to sync: //! * download tasks currently never replace existing local file with metadata file as an exception @@ -129,1441 +114,661 @@ //! * downloads do not contain any actual files to download, so that "external", sync pageserver code is able to schedule the timeline download //! without accessing any extra information about its files. //! +//! FIXME: update this paragraph //! Uploads and downloads sync layer files in arbitrary order, but only after all layer files are synched the local metadada (for download) and remote index part (for upload) are updated, //! to avoid having a corrupt state without the relevant layer files. //! Refer to [`upload`] and [`download`] for more details. //! //! Synchronization never removes any local files from pageserver workdir or remote files from the remote storage, yet there could be overwrites of the same files (index part and metadata file updates, future checksum mismatch fixes). //! NOTE: No real contents or checksum check happens right now and is a subject to improve later. -//! -//! After the whole timeline is downloaded, [`crate::tenant_mgr::apply_timeline_sync_status_updates`] function is used to update pageserver memory stage for the timeline processed. mod delete; mod download; pub mod index; mod upload; -use std::{ - collections::{hash_map, HashMap, HashSet, VecDeque}, - fmt::Debug, - num::{NonZeroU32, NonZeroUsize}, - ops::ControlFlow, - path::{Path, PathBuf}, - sync::{Condvar, Mutex}, -}; +// re-export this +pub use download::list_remote_timelines; -use anyhow::{anyhow, bail, Context}; -use futures::stream::{FuturesUnordered, StreamExt}; -use once_cell::sync::OnceCell; -use remote_storage::GenericRemoteStorage; -use tokio::{ - fs, - time::{Duration, Instant}, -}; -use tracing::*; +use std::collections::{HashMap, VecDeque}; +use std::fmt::Debug; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; + +use anyhow::ensure; +use remote_storage::{DownloadError, GenericRemoteStorage}; +use tokio::runtime::Runtime; +use tracing::{error, info, warn}; + +use utils::lsn::Lsn; + +use self::index::IndexPart; -use self::{ - delete::delete_timeline_layers, - download::{download_timeline_layers, DownloadedTimeline}, - index::{IndexPart, RemoteTimeline, RemoteTimelineIndex}, - upload::{upload_index_part, upload_timeline_layers, UploadedTimeline}, -}; use crate::{ config::PageServerConf, - exponential_backoff, - storage_sync::index::{LayerFileMetadata, RemoteIndex}, + storage_sync::index::{LayerFileMetadata, RelativePath}, task_mgr, task_mgr::TaskKind, task_mgr::BACKGROUND_RUNTIME, tenant::metadata::TimelineMetadata, - tenant_mgr::{attach_local_tenants, TenantAttachData}, -}; -use crate::{ - metrics::{IMAGE_SYNC_TIME, REMAINING_SYNC_ITEMS, REMOTE_INDEX_UPLOAD}, - TenantTimelineValues, + {exponential_backoff, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS}, }; -use crate::metrics::{IMAGE_SYNC_COUNT, IMAGE_SYNC_TIME_HISTOGRAM}; -use utils::id::{TenantId, TenantTimelineId, TimelineId}; +use utils::id::{TenantId, TimelineId}; -use self::download::download_index_parts; -pub use self::download::gather_tenant_timelines_index_parts; +/// A client for accessing a timeline's data in remote storage. +/// +/// This takes care of managing the number of connections, and balancing them +/// across tenants. This also handles retries of failed uploads. +/// +/// Upload and delete requests are ordered so that before a deletion is +/// performed, we wait for all preceding uploads to finish. This ensures sure +/// that if you perform a compaction operation that reshuffles data in layer +/// files, we don't have a transient state where the old files have already been +/// deleted, but new files have not yet been uploaded. +/// +/// Similarly, this enforces an order between index-file uploads, and layer +/// uploads. Before an index-file upload is performed, all preceding layer +/// uploads must be finished. +/// +/// This also maintains a list of remote files, and automatically includes that +/// in the index part file, whenever timeline metadata is uploaded. +/// +/// Downloads are not queued, they are performed immediately. +pub struct RemoteTimelineClient { + conf: &'static PageServerConf, -static SYNC_QUEUE: OnceCell = OnceCell::new(); + runtime: &'static Runtime, -/// A timeline status to share with pageserver's sync counterpart, -/// after comparing local and remote timeline state. -#[derive(Clone, PartialEq, Eq)] -pub enum LocalTimelineInitStatus { - /// The timeline has every remote layer present locally. - /// There could be some layers requiring uploading, - /// but this does not block the timeline from any user interaction. - LocallyComplete(TimelineMetadata), - /// A timeline has some files remotely, that are not present locally and need downloading. - /// Downloading might update timeline's metadata locally and current pageserver logic deals with local layers only, - /// so the data needs to be downloaded first before the timeline can be used. - NeedsSync, + tenant_id: TenantId, + timeline_id: TimelineId, + + upload_queue: Mutex, + + storage_impl: GenericRemoteStorage, } -impl std::fmt::Debug for LocalTimelineInitStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +/// This keeps track of queued and in-progress tasks. +#[derive(Default)] +struct UploadQueue { + /// Counter to assign task IDs + task_counter: u64, + + /// All layer files stored in the remote storage, taking into account all + /// in-progress and queued operations + latest_files: HashMap, + + /// Copy of latest remote metadata file. None if the upload queue hasn't been initialized yet. + /// Can be ahead of what's actually stored in remote storage, if new index file upload + /// has been scheduled but not finished yet. + latest_metadata: Option, + + /// `disk_consistent_lsn` from the last metadata file that was successfully + /// uploaded. None if the upload queue hasn't been initialized yet. + last_uploaded_consistent_lsn: Option, + + // Breakdown of different kinds of tasks currently in-progress + num_inprogress_layer_uploads: usize, + num_inprogress_metadata_uploads: usize, + num_inprogress_deletions: usize, + + /// Tasks that are currently in-progress. In-progress means that a tokio Task + /// has been launched for it. An in-progress task can be busy uploading, but it can + /// also be waiting on the `concurrency_limiter` Semaphore in S3Bucket, or it can + /// be waiting for retry in `exponential_backoff`. + inprogress_tasks: HashMap>, + + /// Queued operations that have not been launched yet. They might depend on previous + /// tasks to finish. For example, metadata upload cannot be performed before all + /// preceding layer file uploads have completed. + queued_operations: VecDeque, +} + +/// An in-progress upload or delete task. +#[derive(Debug)] +struct UploadTask { + /// Unique ID of this task. Used as the key in `inprogress_tasks` above. + task_id: u64, + retries: AtomicU32, + + op: UploadOp, +} + +#[derive(Debug)] +enum UploadOp { + /// Upload a layer file + UploadLayer(PathBuf, LayerFileMetadata), + + /// Upload the metadata file + UploadMetadata(IndexPart, Lsn), + + /// Delete a file. + Delete(PathBuf), + + /// Barrier. When the barrier operation is reached, + Barrier(tokio::sync::watch::Sender<()>), +} + +impl std::fmt::Display for UploadOp { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - Self::LocallyComplete(_) => write!(f, "LocallyComplete"), - Self::NeedsSync => write!(f, "NeedsSync"), + UploadOp::UploadLayer(path, metadata) => write!( + f, + "UploadLayer({}, size={:?})", + path.display(), + metadata.file_size() + ), + UploadOp::UploadMetadata(_, lsn) => write!(f, "UploadMetadata(lsn: {})", lsn), + UploadOp::Delete(path) => write!(f, "Delete({})", path.display()), + UploadOp::Barrier(_) => write!(f, "Barrier"), } } } -/// A structure to combine all synchronization data to share with pageserver after a successful sync loop initialization. -/// Successful initialization includes a case when sync loop is not started, in which case the startup data is returned still, -/// to simplify the received code. -pub struct SyncStartupData { - pub remote_index: RemoteIndex, - pub local_timeline_init_statuses: TenantTimelineValues, -} - -/// Global queue of sync tasks. -/// -/// 'queue' is protected by a mutex, and 'condvar' is used to wait for tasks to arrive. -struct SyncQueue { - max_timelines_per_batch: NonZeroUsize, - - queue: Mutex>, - condvar: Condvar, -} - -impl SyncQueue { - fn new(max_timelines_per_batch: NonZeroUsize) -> Self { - Self { - max_timelines_per_batch, - queue: Mutex::new(VecDeque::new()), - condvar: Condvar::new(), - } - } - - /// Queue a new task - fn push(&self, sync_id: TenantTimelineId, new_task: SyncTask) { - let mut q = self.queue.lock().unwrap(); - - q.push_back((sync_id, new_task)); - if q.len() <= 1 { - self.condvar.notify_one(); - } - } - - /// Fetches a task batch, getting every existing entry from the queue, grouping by timelines and merging the tasks for every timeline. - /// A timeline has to care to not to delete certain layers from the remote storage before the corresponding uploads happen. - /// Other than that, due to "immutable" nature of the layers, the order of their deletion/uploading/downloading does not matter. - /// Hence, we merge the layers together into single task per timeline and run those concurrently (with the deletion happening only after successful uploading). - fn next_task_batch(&self) -> (HashMap, usize) { - // Wait for the first task in blocking fashion - let mut q = self.queue.lock().unwrap(); - while q.is_empty() { - q = self - .condvar - .wait_timeout(q, Duration::from_millis(1000)) - .unwrap() - .0; - - if task_mgr::is_shutdown_requested() { - return (HashMap::new(), q.len()); - } - } - let (first_sync_id, first_task) = q.pop_front().unwrap(); - - let mut timelines_left_to_batch = self.max_timelines_per_batch.get() - 1; - let tasks_to_process = q.len(); - - let mut batches = HashMap::with_capacity(tasks_to_process); - batches.insert(first_sync_id, SyncTaskBatch::new(first_task)); - - let mut tasks_to_reenqueue = Vec::with_capacity(tasks_to_process); - - // Greedily grab as many other tasks that we can. - // Yet do not put all timelines in the batch, but only the first ones that fit the timeline limit. - // Re-enqueue the tasks that don't fit in this batch. - while let Some((sync_id, new_task)) = q.pop_front() { - match batches.entry(sync_id) { - hash_map::Entry::Occupied(mut v) => v.get_mut().add(new_task), - hash_map::Entry::Vacant(v) => { - timelines_left_to_batch = timelines_left_to_batch.saturating_sub(1); - if timelines_left_to_batch == 0 { - tasks_to_reenqueue.push((sync_id, new_task)); - } else { - v.insert(SyncTaskBatch::new(new_task)); - } - } - } - } - - debug!( - "Batched {} timelines, reenqueuing {}", - batches.len(), - tasks_to_reenqueue.len() - ); - for (id, task) in tasks_to_reenqueue { - q.push_back((id, task)); - } - - (batches, q.len()) - } - - #[cfg(test)] - fn len(&self) -> usize { - self.queue.lock().unwrap().len() - } -} - -/// A task to run in the async download/upload loop. -/// Limited by the number of retries, after certain threshold the failing task gets evicted and the timeline disabled. -#[derive(Debug, Clone, PartialEq, Eq)] -enum SyncTask { - /// A checkpoint outcome with possible local file updates that need actualization in the remote storage. - /// Not necessary more fresh than the one already uploaded. - Download(SyncData), - /// A certain amount of image files to download. - Upload(SyncData), - /// Delete remote files. - Delete(SyncData), -} - -/// Stores the data to synd and its retries, to evict the tasks failing to frequently. -#[derive(Debug, Clone, PartialEq, Eq)] -struct SyncData { - retries: u32, - data: T, -} - -impl SyncData { - fn new(retries: u32, data: T) -> Self { - Self { retries, data } - } -} - -impl SyncTask { - fn download(download_task: LayersDownload) -> Self { - Self::Download(SyncData::new(0, download_task)) - } - - fn upload(upload_task: LayersUpload) -> Self { - Self::Upload(SyncData::new(0, upload_task)) - } - - fn delete(delete_task: LayersDeletion) -> Self { - Self::Delete(SyncData::new(0, delete_task)) - } -} - -#[derive(Debug, Default, PartialEq, Eq)] -struct SyncTaskBatch { - upload: Option>, - download: Option>, - delete: Option>, -} - -impl SyncTaskBatch { - fn new(task: SyncTask) -> Self { - let mut new_self = Self::default(); - new_self.add(task); - new_self - } - - fn add(&mut self, task: SyncTask) { - match task { - SyncTask::Download(new_download) => match &mut self.download { - Some(batch_download) => { - batch_download.retries = batch_download.retries.min(new_download.retries); - batch_download - .data - .layers_to_skip - .extend(new_download.data.layers_to_skip.into_iter()); - } - None => self.download = Some(new_download), - }, - SyncTask::Upload(new_upload) => match &mut self.upload { - Some(batch_upload) => { - batch_upload.retries = batch_upload.retries.min(new_upload.retries); - - let batch_data = &mut batch_upload.data; - let new_data = new_upload.data; - batch_data - .layers_to_upload - .extend(new_data.layers_to_upload.into_iter()); - batch_data - .uploaded_layers - .extend(new_data.uploaded_layers.into_iter()); - if batch_data - .metadata - .as_ref() - .map(|meta| meta.disk_consistent_lsn()) - <= new_data - .metadata - .as_ref() - .map(|meta| meta.disk_consistent_lsn()) - { - batch_data.metadata = new_data.metadata; - } - } - None => self.upload = Some(new_upload), - }, - SyncTask::Delete(new_delete) => match &mut self.delete { - Some(batch_delete) => { - batch_delete.retries = batch_delete.retries.min(new_delete.retries); - // Need to reregister deletions, but it's ok to register already deleted files once again, they will be skipped. - batch_delete.data.deletion_registered = batch_delete - .data - .deletion_registered - .min(new_delete.data.deletion_registered); - - // Do not download and upload the layers getting removed in the same batch - if let Some(batch_download) = &mut self.download { - batch_download - .data - .layers_to_skip - .extend(new_delete.data.layers_to_delete.iter().cloned()); - batch_download - .data - .layers_to_skip - .extend(new_delete.data.deleted_layers.iter().cloned()); - } - if let Some(batch_upload) = &mut self.upload { - let not_deleted = |layer: &PathBuf, _: &mut LayerFileMetadata| { - !new_delete.data.layers_to_delete.contains(layer) - && !new_delete.data.deleted_layers.contains(layer) - }; - batch_upload.data.layers_to_upload.retain(not_deleted); - batch_upload.data.uploaded_layers.retain(not_deleted); - } - - batch_delete - .data - .layers_to_delete - .extend(new_delete.data.layers_to_delete.into_iter()); - batch_delete - .data - .deleted_layers - .extend(new_delete.data.deleted_layers.into_iter()); - } - None => self.delete = Some(new_delete), - }, - } - } -} - -/// Local timeline files for upload, appeared after the new checkpoint. -/// Current checkpoint design assumes new files are added only, no deletions or amendment happens. -#[derive(Debug, Clone, PartialEq, Eq)] -struct LayersUpload { - /// Layer file path in the pageserver workdir, that were added for the corresponding checkpoint. - layers_to_upload: HashMap, - /// Already uploaded layers. Used to store the data about the uploads between task retries - /// and to record the data into the remote index after the task got completed or evicted. - uploaded_layers: HashMap, - metadata: Option, -} - -/// A timeline download task. -/// Does not contain the file list to download, to allow other -/// parts of the pageserer code to schedule the task -/// without using the remote index or any other ways to list the remote timeline files. -/// Skips the files that are already downloaded. -#[derive(Debug, Clone, PartialEq, Eq)] -struct LayersDownload { - layers_to_skip: HashSet, - - /// Paths which have been downloaded, and had their metadata verified or generated. +impl RemoteTimelineClient { + /// Initialize the upload queue. This must be called before any uploads can be + /// scheduled. /// - /// Metadata generation happens when upgrading from past version of `IndexPart`. - gathered_metadata: HashMap, -} - -impl LayersDownload { - fn from_skipped_layers(layers_to_skip: HashSet) -> Self { - LayersDownload { - layers_to_skip, - gathered_metadata: HashMap::default(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct LayersDeletion { - layers_to_delete: HashSet, - deleted_layers: HashSet, - /// Pageserver uses [`IndexPart`] as a source of truth for listing the files per timeline. - /// This object gets serialized and placed into the remote storage. - /// So if we manage to update pageserver's [`RemoteIndex`] and update the index part on the remote storage, - /// the corresponding files on S3 won't exist for pageserver albeit being physically present on that remote storage still. - /// Then all that's left is to remove the files from the remote storage, without concerns about consistency. - deletion_registered: bool, -} - -/// Adds the new checkpoint files as an upload sync task to the queue. -/// On task failure, it gets retried again from the start a number of times. -/// -/// Ensure that the loop is started otherwise the task is never processed. -pub fn schedule_layer_upload( - tenant_id: TenantId, - timeline_id: TimelineId, - layers_to_upload: HashMap, - metadata: Option, -) { - let sync_queue = match SYNC_QUEUE.get() { - Some(queue) => queue, - None => { - warn!("Could not send an upload task for tenant {tenant_id}, timeline {timeline_id}"); - return; - } - }; - sync_queue.push( - TenantTimelineId { - tenant_id, - timeline_id, - }, - SyncTask::upload(LayersUpload { - layers_to_upload, - uploaded_layers: HashMap::new(), - metadata, - }), - ); - debug!("Upload task for tenant {tenant_id}, timeline {timeline_id} sent") -} - -/// Adds the new files to delete as a deletion task to the queue. -/// On task failure, it gets retried again from the start a number of times. -/// -/// Ensure that the loop is started otherwise the task is never processed. -pub fn schedule_layer_delete( - tenant_id: TenantId, - timeline_id: TimelineId, - layers_to_delete: HashSet, -) { - let sync_queue = match SYNC_QUEUE.get() { - Some(queue) => queue, - None => { - warn!("Could not send deletion task for tenant {tenant_id}, timeline {timeline_id}"); - return; - } - }; - sync_queue.push( - TenantTimelineId { - tenant_id, - timeline_id, - }, - SyncTask::delete(LayersDeletion { - layers_to_delete, - deleted_layers: HashSet::new(), - deletion_registered: false, - }), - ); - debug!("Deletion task for tenant {tenant_id}, timeline {timeline_id} sent") -} - -/// Requests the download of the entire timeline for a given tenant. -/// No existing local files are currently overwritten, except the metadata file (if its disk_consistent_lsn is less than the downloaded one). -/// The metadata file is always updated last, to avoid inconsistencies. -/// -/// On any failure, the task gets retried, omitting already downloaded layers. -/// -/// Ensure that the loop is started otherwise the task is never processed. -pub fn schedule_layer_download(tenant_id: TenantId, timeline_id: TimelineId) { - debug!("Scheduling layer download for tenant {tenant_id}, timeline {timeline_id}"); - let sync_queue = match SYNC_QUEUE.get() { - Some(queue) => queue, - None => { - warn!("Could not send download task for tenant {tenant_id}, timeline {timeline_id}"); - return; - } - }; - sync_queue.push( - TenantTimelineId { - tenant_id, - timeline_id, - }, - SyncTask::download(LayersDownload::from_skipped_layers(HashSet::new())), - ); - debug!("Download task for tenant {tenant_id}, timeline {timeline_id} sent") -} - -/// Local existing timeline files -/// -/// Values of this type serve different meanings in different contexts. On startup, collected -/// timelines come with the full collected information and when signalling readyness to attach -/// after completed download. After the download the file information is no longer carried, because -/// it is already merged into [`RemoteTimeline`]. -#[derive(Debug)] -pub struct TimelineLocalFiles(TimelineMetadata, HashMap); - -impl TimelineLocalFiles { - pub fn metadata(&self) -> &TimelineMetadata { - &self.0 - } - - /// Called during startup, for all of the local files with full metadata. - pub(crate) fn collected( - metadata: TimelineMetadata, - timeline_files: HashMap, - ) -> TimelineLocalFiles { - TimelineLocalFiles(metadata, timeline_files) - } - - /// Called near the end of tenant initialization, to signal readyness to attach tenants. - pub(crate) fn ready(metadata: TimelineMetadata) -> Self { - TimelineLocalFiles(metadata, HashMap::new()) - } -} - -/// Launch a thread to perform remote storage sync tasks. -/// See module docs for loop step description. -pub fn spawn_storage_sync_task( - conf: &'static PageServerConf, - local_timeline_files: HashMap>, - storage: GenericRemoteStorage, - max_concurrent_timelines_sync: NonZeroUsize, - max_sync_errors: NonZeroU32, -) -> anyhow::Result { - let sync_queue = SyncQueue::new(max_concurrent_timelines_sync); - SYNC_QUEUE - .set(sync_queue) - .map_err(|_queue| anyhow!("Could not initialize sync queue"))?; - let sync_queue = match SYNC_QUEUE.get() { - Some(queue) => queue, - None => bail!("Could not get sync queue during the sync loop step, aborting"), - }; - - // TODO we are able to "attach" empty tenants, but not doing it now since it might require big wait time: - // * we need to list every timeline for tenant on S3, that might be a costly operation - // * we need to download every timeline for the tenant, to activate it in memory - // - // When on-demand download gets merged, we're able to do this fast by storing timeline metadata only. - let mut empty_tenants = TenantTimelineValues::::new(); - let mut keys_for_index_part_downloads = HashSet::new(); - let mut timelines_to_sync = HashMap::new(); - - for (tenant_id, timeline_data) in local_timeline_files { - if timeline_data.is_empty() { - info!("got empty tenant {}", tenant_id); - let _ = empty_tenants.0.entry(tenant_id).or_default(); - } else { - for (timeline_id, timeline_data) in timeline_data { - let id = TenantTimelineId::new(tenant_id, timeline_id); - keys_for_index_part_downloads.insert(id); - timelines_to_sync.insert(id, timeline_data); - } - } - } - - let applicable_index_parts = BACKGROUND_RUNTIME.block_on(download_index_parts( - conf, - &storage, - keys_for_index_part_downloads, - )); - - let remote_index = RemoteIndex::from_parts(conf, applicable_index_parts)?; - - let mut local_timeline_init_statuses = schedule_first_sync_tasks( - &mut BACKGROUND_RUNTIME.block_on(remote_index.write()), - sync_queue, - timelines_to_sync, - ); - local_timeline_init_statuses - .0 - .extend(empty_tenants.0.into_iter()); - - let remote_index_clone = remote_index.clone(); - task_mgr::spawn( - BACKGROUND_RUNTIME.handle(), - TaskKind::StorageSync, - None, - None, - "Remote storage sync task", - false, - async move { - storage_sync_loop( - conf, - (storage, remote_index_clone, sync_queue), - max_sync_errors, - ) - .instrument(info_span!("storage_sync_loop")) - .await; - Ok(()) - }, - ); - Ok(SyncStartupData { - remote_index, - local_timeline_init_statuses, - }) -} - -async fn storage_sync_loop( - conf: &'static PageServerConf, - (storage, index, sync_queue): (GenericRemoteStorage, RemoteIndex, &SyncQueue), - max_sync_errors: NonZeroU32, -) { - info!("Starting remote storage sync loop"); - loop { - let loop_storage = storage.clone(); - - let (batched_tasks, remaining_queue_length) = sync_queue.next_task_batch(); - - if task_mgr::is_shutdown_requested() { - info!("Shutdown requested, stopping"); - break; + /// 'index_part' is the current remote metadata file. + pub fn init_upload_queue(&self, index_part: &IndexPart) -> anyhow::Result<()> { + let mut files = HashMap::new(); + for path in &index_part.timeline_layers { + let layer_metadata = index_part + .layer_metadata + .get(path) + .map(LayerFileMetadata::from) + .unwrap_or(LayerFileMetadata::MISSING); + files.insert(path.clone(), layer_metadata); } - REMAINING_SYNC_ITEMS.set(remaining_queue_length as i64); - if remaining_queue_length > 0 || !batched_tasks.is_empty() { - debug!("Processing tasks for {} timelines in batch, more tasks left to process: {remaining_queue_length}", batched_tasks.len()); - } else { - debug!("No tasks to process"); - continue; - } + let metadata = index_part.parse_metadata()?; - // Concurrently perform all the tasks in the batch - let loop_step = tokio::select! { - step = process_batches( - conf, - max_sync_errors, - loop_storage, - &index, - batched_tasks, - sync_queue, - ) - .instrument(info_span!("storage_sync_loop_step")) => ControlFlow::Continue(step) - , - _ = task_mgr::shutdown_watcher() => ControlFlow::Break(()), - }; - - match loop_step { - ControlFlow::Continue(updated_tenants) => { - if updated_tenants.is_empty() { - debug!("Sync loop step completed, no new tenant states"); - } else { - info!( - "Sync loop step completed, {} new tenant state update(s)", - updated_tenants.len() - ); - let mut timelines_to_attach = HashMap::new(); - let index_accessor = index.read().await; - for tenant_id in updated_tenants { - let tenant_entry = match index_accessor.tenant_entry(&tenant_id) { - Some(tenant_entry) => tenant_entry, - None => { - error!( - "cannot find tenant in remote index for timeline sync update" - ); - continue; - } - }; - - if tenant_entry.has_in_progress_downloads() { - info!("Tenant {tenant_id} has pending timeline downloads, skipping tenant registration"); - continue; - } else { - info!( - "Tenant {tenant_id} download completed. Picking to register in tenant" - ); - // Here we assume that if tenant has no in-progress downloads that - // means that it is the last completed timeline download that triggered - // sync status update. So we look at the index for available timelines - // and register them all at once in a tenant for download - // to be submitted in a single operation to tenant - // so it can apply them at once to internal timeline map. - timelines_to_attach.insert( - tenant_id, - TenantAttachData::Ready( - tenant_entry - .iter() - .map(|(&id, entry)| { - (id, TimelineLocalFiles::ready(entry.metadata.clone())) - }) - .collect(), - ), - ); - } - } - drop(index_accessor); - // Batch timeline download registration to ensure that the external registration code won't block any running tasks before. - attach_local_tenants(conf, &index, timelines_to_attach); - } - } - ControlFlow::Break(()) => { - info!("Shutdown requested, stopping"); - break; - } - } - } -} - -#[derive(Debug)] -enum DownloadStatus { - Downloaded, - Nothing, -} - -#[derive(Debug)] -enum UploadStatus { - Uploaded, - Failed(anyhow::Error), - Nothing, -} - -async fn process_batches( - conf: &'static PageServerConf, - max_sync_errors: NonZeroU32, - storage: GenericRemoteStorage, - index: &RemoteIndex, - batched_tasks: HashMap, - sync_queue: &SyncQueue, -) -> HashSet { - let mut sync_results = batched_tasks - .into_iter() - .map(|(sync_id, batch)| { - let storage = storage.clone(); - let index = index.clone(); - async move { - let state_update = process_sync_task_batch( - conf, - (storage, index, sync_queue), - max_sync_errors, - sync_id, - batch, - ) - .instrument(info_span!("process_sync_task_batch", sync_id = %sync_id)) - .await; - (sync_id, state_update) - } - }) - .collect::>(); - - let mut downloaded_timelines = HashSet::new(); - - while let Some((sync_id, download_marker)) = sync_results.next().await { - debug!( - "Finished storage sync task for sync id {sync_id} download marker {:?}", - download_marker + info!( + "initializing upload queue with disk_consistent_lsn: {}", + metadata.disk_consistent_lsn() ); - if matches!(download_marker, DownloadStatus::Downloaded) { - downloaded_timelines.insert(sync_id.tenant_id); - } - } + let mut upload_queue = self.upload_queue.lock().unwrap(); - downloaded_timelines -} - -async fn process_sync_task_batch( - conf: &'static PageServerConf, - (storage, index, sync_queue): (GenericRemoteStorage, RemoteIndex, &SyncQueue), - max_sync_errors: NonZeroU32, - sync_id: TenantTimelineId, - batch: SyncTaskBatch, -) -> DownloadStatus { - let sync_start = Instant::now(); - let current_remote_timeline = { index.read().await.timeline_entry(&sync_id).cloned() }; - - let upload_data = batch.upload.clone(); - let download_data = batch.download.clone(); - // Run both upload and download tasks concurrently (not in parallel): - // download and upload tasks do not conflict and spoil the pageserver state even if they are executed in parallel. - // Under "spoiling" here means potentially inconsistent layer set that misses some of the layers, declared present - // in local (implicitly, via Lsn values and related memory state) or remote (explicitly via remote layer file paths) metadata. - // When operating in a system without tasks failing over the error threshold, - // current batching and task processing systems aim to update the layer set and metadata files (remote and local), - // without "losing" such layer files. - let (upload_status, download_status) = tokio::join!( - async { - if let Some(upload_data) = upload_data { - let upload_retries = upload_data.retries; - match validate_task_retries(upload_retries, max_sync_errors) - .instrument(info_span!("retries_validation")) - .await - { - ControlFlow::Continue(()) => { - upload_timeline_data( - conf, - (&storage, &index, sync_queue), - current_remote_timeline.as_ref(), - sync_id, - upload_data, - sync_start, - ) - .await - } - ControlFlow::Break(()) => match update_remote_data( - conf, - &storage, - &index, - sync_id, - RemoteDataUpdate::Upload { - uploaded_data: upload_data.data, - upload_failed: true, - }, - ) - .await - { - Ok(()) => UploadStatus::Failed(anyhow::anyhow!( - "Aborted after retries validation, current retries: {upload_retries}, max retries allowed: {max_sync_errors}" - )), - Err(e) => { - error!("Failed to update remote timeline {sync_id}: {e:?}"); - UploadStatus::Failed(e) - } - }, - } - } else { - UploadStatus::Nothing - } - } - .instrument(info_span!("upload_timeline_data")), - async { - if let Some(download_data) = download_data { - match validate_task_retries(download_data.retries, max_sync_errors) - .instrument(info_span!("retries_validation")) - .await - { - ControlFlow::Continue(()) => { - return download_timeline_data( - conf, - (&storage, &index, sync_queue), - current_remote_timeline.as_ref(), - sync_id, - download_data, - sync_start, - ) - .await; - } - ControlFlow::Break(()) => { - index - .write() - .await - .set_awaits_download(&sync_id, false) - .ok(); - } - } - } - DownloadStatus::Nothing - } - .instrument(info_span!("download_timeline_data")), - ); - - if let Some(delete_data) = batch.delete { - match upload_status { - UploadStatus::Uploaded | UploadStatus::Nothing => { - match validate_task_retries(delete_data.retries, max_sync_errors) - .instrument(info_span!("retries_validation")) - .await - { - ControlFlow::Continue(()) => { - delete_timeline_data( - conf, - (&storage, &index, sync_queue), - sync_id, - delete_data, - sync_start, - ) - .instrument(info_span!("delete_timeline_data")) - .await; - } - ControlFlow::Break(()) => { - if let Err(e) = update_remote_data( - conf, - &storage, - &index, - sync_id, - RemoteDataUpdate::Delete(&delete_data.data.deleted_layers), - ) - .await - { - error!("Failed to update remote timeline {sync_id}: {e:?}"); - } - } - } - } - UploadStatus::Failed(e) => { - warn!("Skipping delete task due to failed upload tasks, reenqueuing. Upload data: {:?}, delete data: {delete_data:?}. Upload failure: {e:#}", batch.upload); - sync_queue.push(sync_id, SyncTask::Delete(delete_data)); - } - } - } - - download_status -} - -async fn download_timeline_data( - conf: &'static PageServerConf, - (storage, index, sync_queue): (&GenericRemoteStorage, &RemoteIndex, &SyncQueue), - current_remote_timeline: Option<&RemoteTimeline>, - sync_id: TenantTimelineId, - new_download_data: SyncData, - sync_start: Instant, -) -> DownloadStatus { - static TASK_NAME: &str = "download"; - - match download_timeline_layers( - conf, - storage, - sync_queue, - current_remote_timeline, - sync_id, - new_download_data, - ) - .await - { - DownloadedTimeline::Abort => { - register_sync_status(sync_id, sync_start, TASK_NAME, None); - if let Err(e) = index.write().await.set_awaits_download(&sync_id, false) { - error!("Timeline {sync_id} was expected to be in the remote index after a download attempt, but it's absent: {e:?}"); - } - } - DownloadedTimeline::FailedAndRescheduled => { - register_sync_status(sync_id, sync_start, TASK_NAME, Some(false)); - } - DownloadedTimeline::Successful(mut download_data) => { - match update_local_metadata(conf, sync_id, current_remote_timeline).await { - Ok(()) => { - let mut g = index.write().await; - - match g.set_awaits_download(&sync_id, false) { - Ok(()) => { - let timeline = g - .timeline_entry_mut(&sync_id) - .expect("set_awaits_download verified existence"); - - timeline.merge_metadata_from_downloaded( - &download_data.data.gathered_metadata, - ); - - register_sync_status(sync_id, sync_start, TASK_NAME, Some(true)); - return DownloadStatus::Downloaded; - } - Err(e) => { - error!("Timeline {sync_id} was expected to be in the remote index after a successful download, but it's absent: {e:?}"); - } - }; - } - Err(e) => { - error!("Failed to update local timeline metadata: {e:?}"); - download_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Download(download_data)); - register_sync_status(sync_id, sync_start, TASK_NAME, Some(false)); - } - } - } - } - - DownloadStatus::Nothing -} - -async fn update_local_metadata( - conf: &'static PageServerConf, - sync_id: TenantTimelineId, - remote_timeline: Option<&RemoteTimeline>, -) -> anyhow::Result<()> { - let remote_metadata = match remote_timeline { - Some(timeline) => &timeline.metadata, - None => { - debug!("No remote timeline to update local metadata from, skipping the update"); - return Ok(()); - } - }; - let remote_lsn = remote_metadata.disk_consistent_lsn(); - - let local_metadata_path = conf.metadata_path(sync_id.timeline_id, sync_id.tenant_id); - let local_lsn = if local_metadata_path.exists() { - let local_metadata = read_metadata_file(&local_metadata_path) - .await - .with_context(|| { - format!( - "Failed to load local metadata from path '{}'", - local_metadata_path.display() - ) - })?; - - Some(local_metadata.disk_consistent_lsn()) - } else { - None - }; - - if local_lsn < Some(remote_lsn) { - info!("Updating local timeline metadata from remote timeline: local disk_consistent_lsn={local_lsn:?}, remote disk_consistent_lsn={remote_lsn}"); - // clone because spawn_blocking requires static lifetime - let cloned_metadata = remote_metadata.to_owned(); - let TenantTimelineId { - tenant_id, - timeline_id, - } = sync_id; - tokio::task::spawn_blocking(move || { - crate::tenant::save_metadata(conf, timeline_id, tenant_id, &cloned_metadata, true) - }) - .await - .with_context(|| { - format!( - "failed to join save_metadata task for {}", - local_metadata_path.display() - ) - })? - .with_context(|| { - format!( - "Failed to write remote metadata bytes locally to path '{}'", - local_metadata_path.display() - ) - })?; - } else { - info!("Local metadata at path '{}' has later disk consistent Lsn ({local_lsn:?}) than the remote one ({remote_lsn}), skipping the update", local_metadata_path.display()); - } - - Ok(()) -} - -async fn delete_timeline_data( - conf: &'static PageServerConf, - (storage, index, sync_queue): (&GenericRemoteStorage, &RemoteIndex, &SyncQueue), - sync_id: TenantTimelineId, - mut new_delete_data: SyncData, - sync_start: Instant, -) { - static TASK_NAME: &str = "delete"; - - let timeline_delete = &mut new_delete_data.data; - - if !timeline_delete.deletion_registered { - if let Err(e) = update_remote_data( - conf, - storage, - index, - sync_id, - RemoteDataUpdate::Delete(&timeline_delete.layers_to_delete), - ) - .await + if upload_queue.latest_metadata.is_some() + || upload_queue.last_uploaded_consistent_lsn.is_some() { - error!("Failed to update remote timeline {sync_id}: {e:?}"); - new_delete_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Delete(new_delete_data)); - register_sync_status(sync_id, sync_start, TASK_NAME, Some(false)); - return; + anyhow::bail!("attempted to initialize already initialized upload queue") } - } - timeline_delete.deletion_registered = true; - let sync_status = delete_timeline_layers(storage, sync_queue, sync_id, new_delete_data).await; - register_sync_status(sync_id, sync_start, TASK_NAME, Some(sync_status)); -} + upload_queue.latest_files = files; + upload_queue.latest_metadata = Some(metadata.clone()); + upload_queue.last_uploaded_consistent_lsn = Some(metadata.disk_consistent_lsn()); -async fn read_metadata_file(metadata_path: &Path) -> anyhow::Result { - TimelineMetadata::from_bytes( - &fs::read(metadata_path) - .await - .context("Failed to read local metadata bytes from fs")?, - ) - .context("Failed to parse metadata bytes") -} - -async fn upload_timeline_data( - conf: &'static PageServerConf, - (storage, index, sync_queue): (&GenericRemoteStorage, &RemoteIndex, &SyncQueue), - current_remote_timeline: Option<&RemoteTimeline>, - sync_id: TenantTimelineId, - new_upload_data: SyncData, - sync_start: Instant, -) -> UploadStatus { - static TASK_NAME: &str = "upload"; - let mut uploaded_data = match upload_timeline_layers( - storage, - sync_queue, - current_remote_timeline, - sync_id, - new_upload_data, - ) - .await - { - UploadedTimeline::FailedAndRescheduled(e) => { - register_sync_status(sync_id, sync_start, TASK_NAME, Some(false)); - return UploadStatus::Failed(e); - } - UploadedTimeline::Successful(upload_data) => upload_data, - }; - - match update_remote_data( - conf, - storage, - index, - sync_id, - RemoteDataUpdate::Upload { - uploaded_data: uploaded_data.data.clone(), - upload_failed: false, - }, - ) - .await - { - Ok(()) => { - register_sync_status(sync_id, sync_start, TASK_NAME, Some(true)); - UploadStatus::Uploaded - } - Err(e) => { - error!("Failed to update remote timeline {sync_id}: {e:?}"); - uploaded_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Upload(uploaded_data)); - register_sync_status(sync_id, sync_start, TASK_NAME, Some(false)); - UploadStatus::Failed(e) - } - } -} - -enum RemoteDataUpdate<'a> { - Upload { - uploaded_data: LayersUpload, - upload_failed: bool, - }, - Delete(&'a HashSet), -} - -async fn update_remote_data( - conf: &'static PageServerConf, - storage: &GenericRemoteStorage, - index: &RemoteIndex, - sync_id: TenantTimelineId, - update: RemoteDataUpdate<'_>, -) -> anyhow::Result<()> { - let updated_remote_timeline = { - let mut index_accessor = index.write().await; - - match index_accessor.timeline_entry_mut(&sync_id) { - Some(existing_entry) => { - match update { - RemoteDataUpdate::Upload { - uploaded_data, - upload_failed, - } => { - if let Some(new_metadata) = uploaded_data.metadata.as_ref() { - if existing_entry.metadata.disk_consistent_lsn() - < new_metadata.disk_consistent_lsn() - { - existing_entry.metadata = new_metadata.clone(); - } - } - if upload_failed { - existing_entry.add_upload_failures( - uploaded_data - .layers_to_upload - .iter() - .map(|(k, v)| (k.to_owned(), v.to_owned())), - ); - } else { - existing_entry.add_timeline_layers( - uploaded_data - .uploaded_layers - .iter() - .map(|(k, v)| (k.to_owned(), v.to_owned())), - ); - } - } - RemoteDataUpdate::Delete(layers_to_remove) => { - existing_entry.remove_layers(layers_to_remove) - } - } - existing_entry.clone() - } - None => match update { - RemoteDataUpdate::Upload { - uploaded_data, - upload_failed, - } => { - let new_metadata = match uploaded_data.metadata.as_ref() { - Some(new_metadata) => new_metadata, - None => bail!("For timeline {sync_id} upload, there's no upload metadata and no remote index entry, cannot create a new one"), - }; - let mut new_remote_timeline = RemoteTimeline::new(new_metadata.clone()); - if upload_failed { - new_remote_timeline.add_upload_failures( - uploaded_data - .layers_to_upload - .iter() - .map(|(k, v)| (k.to_owned(), v.to_owned())), - ); - } else { - new_remote_timeline.add_timeline_layers( - uploaded_data - .uploaded_layers - .iter() - .map(|(k, v)| (k.to_owned(), v.to_owned())), - ); - } - - index_accessor.add_timeline_entry(sync_id, new_remote_timeline.clone()); - new_remote_timeline - } - RemoteDataUpdate::Delete(_) => { - warn!("No remote index entry for timeline {sync_id}, skipping deletion"); - return Ok(()); - } - }, - } - }; - - let timeline_path = conf.timeline_path(&sync_id.timeline_id, &sync_id.tenant_id); - let new_index_part = - IndexPart::from_remote_timeline(&timeline_path, updated_remote_timeline) - .context("Failed to create an index part from the updated remote timeline")?; - - debug!("Uploading remote index for the timeline"); - REMOTE_INDEX_UPLOAD - .with_label_values(&[ - &sync_id.tenant_id.to_string(), - &sync_id.timeline_id.to_string(), - ]) - .inc(); - - upload_index_part(conf, storage, sync_id, new_index_part) - .await - .context("Failed to upload new index part") -} - -async fn validate_task_retries( - current_attempt: u32, - max_sync_errors: NonZeroU32, -) -> ControlFlow<(), ()> { - let max_sync_errors = max_sync_errors.get(); - if current_attempt >= max_sync_errors { - return ControlFlow::Break(()); + Ok(()) } - exponential_backoff(current_attempt, 1.0, 30.0).await; - ControlFlow::Continue(()) -} + /// Like init_upload_queue, but the list of files is initially empty, and + /// metadata is passed as a TimelineMetadata. This is used when creating a + /// new timeline. + pub fn init_upload_queue_empty(&self, metadata: &TimelineMetadata) -> anyhow::Result<()> { + info!( + "initializing upload queue for new timeline with disk_consistent_lsn: {}", + metadata.disk_consistent_lsn() + ); -fn schedule_first_sync_tasks( - index: &mut RemoteTimelineIndex, - sync_queue: &SyncQueue, - local_timeline_files: HashMap, -) -> TenantTimelineValues { - let mut local_timeline_init_statuses = TenantTimelineValues::new(); - - let mut new_sync_tasks = VecDeque::with_capacity(local_timeline_files.len()); - - for (sync_id, local_timeline) in local_timeline_files { - let TimelineLocalFiles(local_metadata, local_files) = local_timeline; - match index.timeline_entry_mut(&sync_id) { - Some(remote_timeline) => { - let (timeline_status, awaits_download) = compare_local_and_remote_timeline( - &mut new_sync_tasks, - sync_id, - local_metadata, - local_files, - remote_timeline, - ); - match local_timeline_init_statuses - .0 - .entry(sync_id.tenant_id) - .or_default() - .entry(sync_id.timeline_id) - { - hash_map::Entry::Occupied(mut o) => { - { - // defensive check - warn!( - "Overwriting timeline init sync status. Status {timeline_status:?}, timeline {}", - sync_id.timeline_id - ); - } - o.insert(timeline_status); - } - hash_map::Entry::Vacant(v) => { - v.insert(timeline_status); - } - } - - remote_timeline.awaits_download = awaits_download; - } - None => { - // TODO (rodionov) does this mean that we've crashed during tenant creation? - // is it safe to upload this checkpoint? could it be half broken? - warn!( - "marking {} as locally complete, while it doesnt exist in remote index", - sync_id - ); - new_sync_tasks.push_back(( - sync_id, - SyncTask::upload(LayersUpload { - layers_to_upload: local_files, - uploaded_layers: HashMap::new(), - metadata: Some(local_metadata.clone()), - }), - )); - local_timeline_init_statuses - .0 - .entry(sync_id.tenant_id) - .or_default() - .insert( - sync_id.timeline_id, - LocalTimelineInitStatus::LocallyComplete(local_metadata), - ); - } + let mut upload_queue = self.upload_queue.lock().unwrap(); + if upload_queue.latest_metadata.is_some() + || upload_queue.last_uploaded_consistent_lsn.is_some() + { + anyhow::bail!("attempted to initialize already initialized upload queue") } + + upload_queue.latest_metadata = Some(metadata.clone()); + upload_queue.last_uploaded_consistent_lsn = Some(metadata.disk_consistent_lsn()); + Ok(()) } - new_sync_tasks.into_iter().for_each(|(sync_id, task)| { - sync_queue.push(sync_id, task); - }); - local_timeline_init_statuses -} + pub fn last_uploaded_consistent_lsn(&self) -> Option { + let upload_queue = self.upload_queue.lock().unwrap(); + upload_queue.last_uploaded_consistent_lsn + } -/// bool in return value stands for awaits_download -fn compare_local_and_remote_timeline( - new_sync_tasks: &mut VecDeque<(TenantTimelineId, SyncTask)>, - sync_id: TenantTimelineId, - local_metadata: TimelineMetadata, - local_files: HashMap, - remote_entry: &RemoteTimeline, -) -> (LocalTimelineInitStatus, bool) { - let _entered = info_span!("compare_local_and_remote_timeline", sync_id = %sync_id).entered(); + // + // Download operations. + // + // These don't use the per-timeline queue. They do use the global semaphore in + // S3Bucket, to limit the total number of concurrent operations, though. + // - let needed_to_download_files = remote_entry - .stored_files() - .iter() - .filter_map(|(layer_file, remote_metadata)| { - if let Some(local_metadata) = local_files.get(layer_file) { - match (remote_metadata.file_size(), local_metadata.file_size()) { - (Some(x), Some(y)) if x == y => { None }, - (None, Some(_)) => { - // upgrading from an earlier IndexPart without metadata - None - }, - _ => { - // having to deal with other than (Some(x), Some(y)) where x != y here is a - // bummer, but see #2582 and #2610 for attempts and discussion. - warn!("Redownloading locally existing {layer_file:?} due to size mismatch, size on index: {:?}, on disk: {:?}", remote_metadata.file_size(), local_metadata.file_size()); - Some(layer_file) - }, - } - } else { - // doesn't exist locally - Some(layer_file) - } - }) - .collect::>(); - - let (initial_timeline_status, awaits_download) = if !needed_to_download_files.is_empty() { - new_sync_tasks.push_back(( - sync_id, - SyncTask::download(LayersDownload::from_skipped_layers( - local_files - .keys() - .filter(|path| !needed_to_download_files.contains(path)) - .cloned() - .collect(), - )), - )); - info!("NeedsSync"); - (LocalTimelineInitStatus::NeedsSync, true) - // we do not need to manipulate with remote consistent lsn here - // because it will be updated when sync will be completed - } else { - info!("LocallyComplete"); - ( - LocalTimelineInitStatus::LocallyComplete(local_metadata.clone()), - false, + /// Download index file + pub async fn download_index_file(&self) -> Result { + download::download_index_part( + self.conf, + &self.storage_impl, + self.tenant_id, + self.timeline_id, ) - }; - - let layers_to_upload = local_files - .iter() - .filter_map(|(local_file, metadata)| { - if !remote_entry.stored_files().contains_key(local_file) { - Some((local_file.to_owned(), metadata.to_owned())) - } else { - None - } - }) - .collect::>(); - - if !layers_to_upload.is_empty() { - new_sync_tasks.push_back(( - sync_id, - SyncTask::upload(LayersUpload { - layers_to_upload, - uploaded_layers: HashMap::new(), - metadata: Some(local_metadata), - }), - )); - // Note that status here doesn't change. + .await } - (initial_timeline_status, awaits_download) -} - -fn register_sync_status( - sync_id: TenantTimelineId, - sync_start: Instant, - sync_name: &str, - sync_status: Option, -) { - let secs_elapsed = sync_start.elapsed().as_secs_f64(); - debug!("Processed a sync task in {secs_elapsed:.2} seconds"); - - let tenant_id = sync_id.tenant_id.to_string(); - let timeline_id = sync_id.timeline_id.to_string(); - - let sync_status = match sync_status { - Some(true) => "success", - Some(false) => "failure", - None => "abort", - }; - - IMAGE_SYNC_TIME_HISTOGRAM - .with_label_values(&[sync_name, sync_status]) - .observe(secs_elapsed); - IMAGE_SYNC_TIME - .with_label_values(&[&tenant_id, &timeline_id]) - .add(secs_elapsed); - IMAGE_SYNC_COUNT - .with_label_values(&[&tenant_id, &timeline_id, sync_name, sync_status]) - .inc(); -} - -#[cfg(test)] -mod test_utils { - use utils::lsn::Lsn; - - use crate::tenant::harness::TenantHarness; - - use super::*; - - pub(super) async fn create_local_timeline( - harness: &TenantHarness<'_>, - timeline_id: TimelineId, - filenames: &[&str], - metadata: TimelineMetadata, - ) -> anyhow::Result { - let timeline_path = harness.timeline_path(&timeline_id); - fs::create_dir_all(&timeline_path).await?; - - let mut layers_to_upload = HashMap::with_capacity(filenames.len()); - for &file in filenames { - let file_path = timeline_path.join(file); - fs::write(&file_path, dummy_contents(file).into_bytes()).await?; - let metadata = LayerFileMetadata::new(file_path.metadata()?.len()); - layers_to_upload.insert(file_path, metadata); - } - - fs::write( - harness.conf.metadata_path(timeline_id, harness.tenant_id), - metadata.to_bytes()?, + /// Download a (layer) file from `path`, into local filesystem. + /// + /// 'layer_metadata' is the metadata from the remote index file. + pub async fn download_layer_file( + &self, + path: &RelativePath, + layer_metadata: &LayerFileMetadata, + ) -> anyhow::Result<()> { + let downloaded_size = download::download_layer_file( + self.conf, + &self.storage_impl, + self.tenant_id, + self.timeline_id, + path, + layer_metadata, ) .await?; - Ok(LayersUpload { - layers_to_upload, - uploaded_layers: HashMap::new(), - metadata: Some(metadata), - }) + // Update the metadata for given layer file. The remote index file + // might be missing some information for the file; this allows us + // to fill in the missing details. + if layer_metadata.file_size().is_none() { + let new_metadata = LayerFileMetadata::new(downloaded_size); + let mut upload_queue = self.upload_queue.lock().unwrap(); + if let Some(upgraded) = upload_queue.latest_files.get_mut(path) { + upgraded.merge(&new_metadata); + } else { + // The file should exist, since we just downloaded it. + warn!( + "downloaded file {:?} not found in local copy of the index file", + path + ); + } + } + Ok(()) } - pub(super) fn dummy_contents(name: &str) -> String { - format!("contents for {name}") + // + // Upload operations. + // + + /// + /// Launch an index-file upload operation in the background. + /// + /// The upload will be added to the queue immediately, but it + /// won't be performed until all previosuly scheduled layer file + /// upload operations have completed successfully. This is to + /// ensure that when the index file claims that layers X, Y and Z + /// exist in remote storage, they really do. + pub fn schedule_index_upload( + self: &Arc, + metadata: Option<&TimelineMetadata>, + ) -> anyhow::Result<()> { + let mut upload_queue = self.upload_queue.lock().unwrap(); + ensure!( + upload_queue.latest_metadata.is_some(), + "upload queue not initialized" + ); + + if let Some(new_metadata) = metadata { + upload_queue.latest_metadata = Some(new_metadata.clone()); + } + + let disk_consistent_lsn = upload_queue + .latest_metadata + .as_ref() + .unwrap() + .disk_consistent_lsn(); + + let index_part = IndexPart::new( + upload_queue.latest_files.clone(), + disk_consistent_lsn, + upload_queue.latest_metadata.as_ref().unwrap().to_bytes()?, + ); + upload_queue + .queued_operations + .push_back(UploadOp::UploadMetadata(index_part, disk_consistent_lsn)); + + info!( + "scheduled metadata upload with {} files", + upload_queue.latest_files.len() + ); + + // Launch the task immediately, if possible + self.launch_queued_tasks(upload_queue); + + Ok(()) + } + + /// + /// Launch an upload operation in the background. + /// + pub fn schedule_layer_file_upload( + self: &Arc, + path: &Path, + layer_metadata: &LayerFileMetadata, + ) -> anyhow::Result<()> { + let mut upload_queue = self.upload_queue.lock().unwrap(); + ensure!( + upload_queue.last_uploaded_consistent_lsn.is_some(), + "upload queue not initialized" + ); + // The file size can be missing for files that were created before we tracked that + // in the metadata, but it should be present for any new files we create. + ensure!( + layer_metadata.file_size().is_some(), + "file size not initialized in metadata" + ); + + let relative_path = RelativePath::from_local_path( + &self.conf.timeline_path(&self.timeline_id, &self.tenant_id), + path, + )?; + + upload_queue + .latest_files + .insert(relative_path, layer_metadata.clone()); + + upload_queue + .queued_operations + .push_back(UploadOp::UploadLayer( + PathBuf::from(path), + layer_metadata.clone(), + )); + + info!("scheduled layer file upload {}", path.display()); + + // Launch the task immediately, if possible + self.launch_queued_tasks(upload_queue); + Ok(()) + } + + /// + /// Launch a delete operation in the background. + /// + /// The deletion won't actually be performed, until all preceding + /// upload operations have completed succesfully. + pub fn schedule_layer_file_deletion(self: &Arc, paths: &[PathBuf]) -> anyhow::Result<()> { + let mut upload_queue = self.upload_queue.lock().unwrap(); + ensure!( + upload_queue.latest_metadata.is_some(), + "upload queue not initialized" + ); + + // Update the remote index file, removing the to-be-deleted files from the index, + // before deleting the actual files. + for path in paths { + let relative_path = RelativePath::from_local_path( + &self.conf.timeline_path(&self.timeline_id, &self.tenant_id), + path, + )?; + upload_queue.latest_files.remove(&relative_path); + } + let disk_consistent_lsn = upload_queue + .latest_metadata + .as_ref() + .unwrap() + .disk_consistent_lsn(); + let index_part = IndexPart::new( + upload_queue.latest_files.clone(), + disk_consistent_lsn, + upload_queue.latest_metadata.as_ref().unwrap().to_bytes()?, + ); + upload_queue + .queued_operations + .push_back(UploadOp::UploadMetadata(index_part, disk_consistent_lsn)); + + // schedule the actual deletions + for path in paths { + upload_queue + .queued_operations + .push_back(UploadOp::Delete(PathBuf::from(path))); + info!("scheduled layer file deletion {}", path.display()); + } + + // Launch the tasks immediately, if possible + self.launch_queued_tasks(upload_queue); + Ok(()) + } + + /// + /// Wait for all previously scheduled uploads/deletions to complete + /// + pub async fn wait_completion(self: &Arc) -> anyhow::Result<()> { + let (sender, mut receiver) = tokio::sync::watch::channel(()); + let barrier_op = UploadOp::Barrier(sender); + + { + let mut upload_queue = self.upload_queue.lock().unwrap(); + ensure!( + upload_queue.latest_metadata.is_some(), + "upload queue not initialized" + ); + + upload_queue.queued_operations.push_back(barrier_op); + + // Launch the task immediately, if possible + self.launch_queued_tasks(upload_queue); + } + + receiver.changed().await?; + Ok(()) + } + + /// + /// Pick next tasks from the queue, and start as many of them as possible without violating + /// the ordering constraints. + /// + /// The caller needs to already hold the `upload_queue` lock. + fn launch_queued_tasks(self: &Arc, mut upload_queue: MutexGuard) { + while let Some(next_op) = upload_queue.queued_operations.front() { + // Can we run this task now? + let can_run_now = match next_op { + UploadOp::UploadLayer(_, _) => { + // Can always be scheduled. + true + } + UploadOp::UploadMetadata(_, _) => { + // These can only be performed after all the preceding operations + // have finished. + upload_queue.inprogress_tasks.is_empty() + } + UploadOp::Delete(_) => { + // Wait for preceding uploads to finish. Concurrent deletions are OK, though. + upload_queue.num_inprogress_deletions == upload_queue.inprogress_tasks.len() + } + + UploadOp::Barrier(_) => upload_queue.inprogress_tasks.is_empty(), + }; + + // If we cannot launch this task, don't look any further. + // + // In some cases, we could let some non-frontmost tasks to "jump the queue" and launch + // them now, but we don't try to do that currently. For example, if the frontmost task + // is an index-file upload that cannot proceed until preceding uploads have finished, we + // could still start layer uploads that were scheduled later. + if !can_run_now { + break; + } + + // We can launch this task. Remove it from the queue first. + let next_op = upload_queue.queued_operations.pop_front().unwrap(); + + info!("starting op: {}", next_op); + + // Update the counters + match next_op { + UploadOp::UploadLayer(_, _) => { + upload_queue.num_inprogress_layer_uploads += 1; + } + UploadOp::UploadMetadata(_, _) => { + upload_queue.num_inprogress_metadata_uploads += 1; + } + UploadOp::Delete(_) => { + upload_queue.num_inprogress_deletions += 1; + } + UploadOp::Barrier(sender) => { + sender.send_replace(()); + continue; + } + }; + + // Assign unique ID to this task + upload_queue.task_counter += 1; + let task_id = upload_queue.task_counter; + + // Add it to the in-progress map + let task = Arc::new(UploadTask { + task_id, + op: next_op, + retries: AtomicU32::new(0), + }); + upload_queue + .inprogress_tasks + .insert(task.task_id, Arc::clone(&task)); + + // Spawn task to perform the task + let self_rc = Arc::clone(self); + task_mgr::spawn( + self.runtime.handle(), + TaskKind::RemoteUploadTask, + Some(self.tenant_id), + Some(self.timeline_id), + "remote upload", + false, + async move { + self_rc.perform_upload_task(task).await; + Ok(()) + }, + ); + + // Loop back to process next task + } + } + + /// + /// Perform an upload task. + /// + /// The task is in the `inprogress_tasks` list. This function will try to + /// execute it, retrying forever. On successful completion, the task is + /// removed it from the `inprogress_tasks` list, and any next task(s) in the + /// queue that were waiting by the completion are launched. + /// + async fn perform_upload_task(self: &Arc, task: Arc) { + // Loop to retry until it completes. + loop { + let upload_result: anyhow::Result<()> = match task.op { + UploadOp::UploadLayer(ref path, ref layer_metadata) => { + upload::upload_timeline_layer(&self.storage_impl, path, layer_metadata).await + } + UploadOp::UploadMetadata(ref index_part, _lsn) => { + upload::upload_index_part( + self.conf, + &self.storage_impl, + self.tenant_id, + self.timeline_id, + index_part, + ) + .await + } + UploadOp::Delete(ref path) => delete::delete_layer(&self.storage_impl, path).await, + UploadOp::Barrier(_) => { + // unreachable. Barrier operations are handled synchronously in + // launch_queued_tasks + warn!("unexpected Barrier operation in perform_upload_task"); + break; + } + }; + + match upload_result { + Ok(()) => { + break; + } + Err(e) => { + let retries = task.retries.fetch_add(1, Ordering::SeqCst); + + error!( + "failed to perform remote task {}, will retry (attempt {}): {:?}", + task.op, retries, e + ); + + exponential_backoff( + retries, + DEFAULT_BASE_BACKOFF_SECONDS, + DEFAULT_MAX_BACKOFF_SECONDS, + ) + .await; + } + } + } + + let retries = task.retries.load(Ordering::SeqCst); + if retries > 0 { + info!( + "remote task {} completed successfully after {} retries", + task.op, retries + ); + } else { + info!("remote task {} completed successfully", task.op); + } + + // The task has completed succesfully. Remove it from the in-progress list. + { + let mut upload_queue = self.upload_queue.lock().unwrap(); + upload_queue.inprogress_tasks.remove(&task.task_id); + + match task.op { + UploadOp::UploadLayer(_, _) => { + upload_queue.num_inprogress_layer_uploads -= 1; + } + UploadOp::UploadMetadata(_, lsn) => { + upload_queue.num_inprogress_metadata_uploads -= 1; + upload_queue.last_uploaded_consistent_lsn = Some(lsn); + } + UploadOp::Delete(_) => { + upload_queue.num_inprogress_deletions -= 1; + } + UploadOp::Barrier(_) => unreachable!(), + }; + + // Launch any queued tasks that were unblocked by this one. + self.launch_queued_tasks(upload_queue); + } + } +} + +/// +/// Create a remote storage client for given timeline +/// +/// Note: the caller must initialize the upload queue before any uploads can be scheduled, +/// by calling init_upload_queue. +/// +pub fn create_remote_timeline_client( + remote_storage: GenericRemoteStorage, + conf: &'static PageServerConf, + tenant_id: TenantId, + timeline_id: TimelineId, +) -> anyhow::Result { + Ok(RemoteTimelineClient { + conf, + runtime: &BACKGROUND_RUNTIME, + tenant_id, + timeline_id, + storage_impl: remote_storage, + upload_queue: Mutex::new(UploadQueue::default()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tenant::harness::{TenantHarness, TIMELINE_ID}; + use remote_storage::{RemoteStorageConfig, RemoteStorageKind}; + use std::collections::HashSet; + use utils::lsn::Lsn; + + pub(super) fn dummy_contents(name: &str) -> Vec { + format!("contents for {name}").into() } pub(super) fn dummy_metadata(disk_consistent_lsn: Lsn) -> TimelineMetadata { - TimelineMetadata::new( + let metadata = TimelineMetadata::new( disk_consistent_lsn, None, None, @@ -1573,335 +778,180 @@ mod test_utils { // Any version will do // but it should be consistent with the one in the tests crate::DEFAULT_PG_VERSION, - ) + ); + + // go through serialize + deserialize to fix the header, including checksum + TimelineMetadata::from_bytes(&metadata.to_bytes().unwrap()).unwrap() } -} -#[cfg(test)] -mod tests { - use super::test_utils::dummy_metadata; - use crate::tenant::harness::TIMELINE_ID; - use hex_literal::hex; - use utils::lsn::Lsn; + fn assert_file_list(a: &HashSet, b: &[&str]) { + let xx = PathBuf::from(""); + let mut avec: Vec = a + .iter() + .map(|x| x.to_local_path(&xx).to_string_lossy().into()) + .collect(); + avec.sort(); - use super::*; + let mut bvec = b.to_owned(); + bvec.sort_unstable(); - const TEST_SYNC_ID: TenantTimelineId = TenantTimelineId { - tenant_id: TenantId::from_array(hex!("11223344556677881122334455667788")), - timeline_id: TIMELINE_ID, - }; + assert_eq!(avec, bvec); + } - #[tokio::test] - async fn separate_task_ids_batch() { - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - assert_eq!(sync_queue.len(), 0); + fn assert_remote_files(expected: &[&str], remote_path: &Path) { + let mut expected: Vec = expected.iter().map(|x| String::from(*x)).collect(); + expected.sort(); - let sync_id_2 = TenantTimelineId { - tenant_id: TenantId::from_array(hex!("22223344556677881122334455667788")), - timeline_id: TIMELINE_ID, + let mut found: Vec = Vec::new(); + for entry in std::fs::read_dir(remote_path).unwrap().flatten() { + let entry_name = entry.file_name(); + let fname = entry_name.to_str().unwrap(); + found.push(String::from(fname)); + } + found.sort(); + + assert_eq!(found, expected); + } + + // Test scheduling + #[test] + fn upload_scheduling() -> anyhow::Result<()> { + let harness = TenantHarness::create("upload_scheduling")?; + let timeline_path = harness.timeline_path(&TIMELINE_ID); + std::fs::create_dir_all(&timeline_path)?; + + let remote_fs_dir = harness.conf.workdir.join("remote_fs"); + std::fs::create_dir_all(remote_fs_dir)?; + let remote_fs_dir = std::fs::canonicalize(harness.conf.workdir.join("remote_fs"))?; + + let storage_config = RemoteStorageConfig { + max_concurrent_syncs: std::num::NonZeroUsize::new( + remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS, + ) + .unwrap(), + max_sync_errors: std::num::NonZeroU32::new( + remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS, + ) + .unwrap(), + storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; - let sync_id_3 = TenantTimelineId { - tenant_id: TenantId::from_array(hex!("33223344556677881122334455667788")), - timeline_id: TIMELINE_ID, - }; - assert!(sync_id_2 != TEST_SYNC_ID); - assert!(sync_id_2 != sync_id_3); - assert!(sync_id_3 != TEST_SYNC_ID); - let download_task = - SyncTask::download(LayersDownload::from_skipped_layers(HashSet::from([ - PathBuf::from("sk"), - ]))); - let upload_task = SyncTask::upload(LayersUpload { - layers_to_upload: HashMap::from([(PathBuf::from("up"), LayerFileMetadata::new(123))]), - uploaded_layers: HashMap::from([(PathBuf::from("upl"), LayerFileMetadata::new(123))]), - metadata: Some(dummy_metadata(Lsn(2))), - }); - let delete_task = SyncTask::delete(LayersDeletion { - layers_to_delete: HashSet::from([PathBuf::from("de")]), - deleted_layers: HashSet::from([PathBuf::from("del")]), - deletion_registered: false, + // Use a current-thread runtime in the test + let runtime = Box::leak(Box::new( + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + )); + let _entered = runtime.enter(); + + // Test outline: + // + // Schedule upload of a bunch of layers. Check that they are started immediately, not queued + // Schedule upload of index. Check that it is queued + // let the layer file uploads finish. Check that the index-upload is now started + // let the index-upload finish. + // + // Download back the index.json. Check that the list of files is correct + // + // Schedule upload. Schedule deletion. Check that the deletion is queued + // let upload finish. Check that deletion is now started + // Schedule another deletion. Check that it's launched immediately. + // Schedule index upload. Check that it's queued + + println!("workdir: {}", harness.conf.workdir.display()); + + let storage_impl = + GenericRemoteStorage::from_config(harness.conf.workdir.clone(), &storage_config)?; + let client = Arc::new(RemoteTimelineClient { + conf: harness.conf, + runtime, + tenant_id: harness.tenant_id, + timeline_id: TIMELINE_ID, + storage_impl, + upload_queue: Mutex::new(UploadQueue::default()), }); - sync_queue.push(TEST_SYNC_ID, download_task.clone()); - sync_queue.push(sync_id_2, upload_task.clone()); - sync_queue.push(sync_id_3, delete_task.clone()); + let remote_timeline_dir = + remote_fs_dir.join(timeline_path.strip_prefix(&harness.conf.workdir)?); + println!("remote_timeline_dir: {}", remote_timeline_dir.display()); - let submitted_tasks_count = sync_queue.len(); - assert_eq!(submitted_tasks_count, 3); - let (mut batch, _) = sync_queue.next_task_batch(); - assert_eq!( - batch.len(), - submitted_tasks_count, - "Batch should consist of all tasks submitted" - ); + let metadata = dummy_metadata(Lsn(0x10)); + client.init_upload_queue_empty(&metadata)?; - assert_eq!( - Some(SyncTaskBatch::new(download_task)), - batch.remove(&TEST_SYNC_ID) - ); - assert_eq!( - Some(SyncTaskBatch::new(upload_task)), - batch.remove(&sync_id_2) - ); - assert_eq!( - Some(SyncTaskBatch::new(delete_task)), - batch.remove(&sync_id_3) - ); + // Create a couple of dummy files, schedule upload for them + let content_foo = dummy_contents("foo"); + let content_bar = dummy_contents("bar"); + std::fs::write(timeline_path.join("foo"), &content_foo)?; + std::fs::write(timeline_path.join("bar"), &content_bar)?; - assert!(batch.is_empty(), "Should check all batch tasks"); - assert_eq!(sync_queue.len(), 0); - } + client.schedule_layer_file_upload( + &timeline_path.join("foo"), + &LayerFileMetadata::new(content_foo.len() as u64), + )?; + client.schedule_layer_file_upload( + &timeline_path.join("bar"), + &LayerFileMetadata::new(content_bar.len() as u64), + )?; - #[tokio::test] - async fn same_task_id_separate_tasks_batch() { - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - assert_eq!(sync_queue.len(), 0); - - let download = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk")])); - let upload = LayersUpload { - layers_to_upload: HashMap::from([(PathBuf::from("up"), LayerFileMetadata::new(123))]), - uploaded_layers: HashMap::from([(PathBuf::from("upl"), LayerFileMetadata::new(123))]), - metadata: Some(dummy_metadata(Lsn(2))), - }; - let delete = LayersDeletion { - layers_to_delete: HashSet::from([PathBuf::from("de")]), - deleted_layers: HashSet::from([PathBuf::from("del")]), - deletion_registered: false, - }; - - sync_queue.push(TEST_SYNC_ID, SyncTask::download(download.clone())); - sync_queue.push(TEST_SYNC_ID, SyncTask::upload(upload.clone())); - sync_queue.push(TEST_SYNC_ID, SyncTask::delete(delete.clone())); - - let submitted_tasks_count = sync_queue.len(); - assert_eq!(submitted_tasks_count, 3); - let (mut batch, _) = sync_queue.next_task_batch(); - assert_eq!( - batch.len(), - 1, - "Queue should have one batch merged from 3 sync tasks of the same user" - ); - - assert_eq!( - Some(SyncTaskBatch { - upload: Some(SyncData { - retries: 0, - data: upload - }), - download: Some(SyncData { - retries: 0, - data: download - }), - delete: Some(SyncData { - retries: 0, - data: delete - }), - }), - batch.remove(&TEST_SYNC_ID), - "Should have one batch containing all tasks unchanged" - ); - - assert!(batch.is_empty(), "Should check all batch tasks"); - assert_eq!(sync_queue.len(), 0); - } - - #[tokio::test] - async fn same_task_id_same_tasks_batch() { - let sync_queue = SyncQueue::new(NonZeroUsize::new(1).unwrap()); - let download_1 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk1")])); - let download_2 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk2")])); - let download_3 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk3")])); - let download_4 = LayersDownload::from_skipped_layers(HashSet::from([PathBuf::from("sk4")])); - - let sync_id_2 = TenantTimelineId { - tenant_id: TenantId::from_array(hex!("22223344556677881122334455667788")), - timeline_id: TIMELINE_ID, - }; - assert!(sync_id_2 != TEST_SYNC_ID); - - sync_queue.push(TEST_SYNC_ID, SyncTask::download(download_1.clone())); - sync_queue.push(TEST_SYNC_ID, SyncTask::download(download_2.clone())); - sync_queue.push(sync_id_2, SyncTask::download(download_3)); - sync_queue.push(TEST_SYNC_ID, SyncTask::download(download_4.clone())); - assert_eq!(sync_queue.len(), 4); - - let (mut smallest_batch, _) = sync_queue.next_task_batch(); - assert_eq!( - smallest_batch.len(), - 1, - "Queue should have one batch merged from the all sync tasks, but not the other user's task" - ); - assert_eq!( - Some(SyncTaskBatch { - download: Some(SyncData { - retries: 0, - data: LayersDownload::from_skipped_layers( - { - let mut set = HashSet::new(); - set.extend(download_1.layers_to_skip.into_iter()); - set.extend(download_2.layers_to_skip.into_iter()); - set.extend(download_4.layers_to_skip.into_iter()); - set - }, - ) - }), - upload: None, - delete: None, - }), - smallest_batch.remove(&TEST_SYNC_ID), - "Should have one batch containing all tasks merged for the tenant first appeared in the batch" - ); - - assert!(smallest_batch.is_empty(), "Should check all batch tasks"); - assert_eq!( - sync_queue.len(), - 1, - "Should have one task left out of the batch" - ); - } - - mod local_and_remote_comparisons { - use super::*; - - #[test] - fn ready() { - let mut new_sync_tasks = VecDeque::default(); - let sync_id = TenantTimelineId::generate(); - let local_metadata = dummy_metadata(0x02.into()); - let local_files = - HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]); - let mut remote_entry = RemoteTimeline::new(local_metadata.clone()); - remote_entry - .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]); - - let (status, sync_needed) = compare_local_and_remote_timeline( - &mut new_sync_tasks, - sync_id, - local_metadata.clone(), - local_files, - &remote_entry, - ); - - assert_eq!( - status, - LocalTimelineInitStatus::LocallyComplete(local_metadata) - ); - assert!(!sync_needed); - - assert!(new_sync_tasks.is_empty(), "{:?}", new_sync_tasks); + // Check that they are started immediately, not queued + { + let upload_queue = client.upload_queue.lock().unwrap(); + assert!(upload_queue.queued_operations.is_empty()); + assert!(upload_queue.inprogress_tasks.len() == 2); + assert!(upload_queue.num_inprogress_layer_uploads == 2); } - #[test] - fn needs_download() { - let mut new_sync_tasks = VecDeque::default(); - let sync_id = TenantTimelineId::generate(); - let local_metadata = dummy_metadata(0x02.into()); - let local_files = HashMap::default(); - let mut remote_entry = RemoteTimeline::new(local_metadata.clone()); - remote_entry - .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]); - - let (status, sync_needed) = compare_local_and_remote_timeline( - &mut new_sync_tasks, - sync_id, - local_metadata, - local_files.clone(), - &remote_entry, - ); - - assert_eq!(status, LocalTimelineInitStatus::NeedsSync); - assert!(sync_needed); - - let new_sync_tasks = new_sync_tasks.into_iter().collect::>(); - - assert_eq!( - &new_sync_tasks, - &[( - sync_id, - SyncTask::download(LayersDownload::from_skipped_layers( - local_files.keys().cloned().collect() - )) - )] - ); + // Schedule upload of index. Check that it is queued + let metadata = dummy_metadata(Lsn(0x20)); + client.schedule_index_upload(Some(&metadata))?; + { + let upload_queue = client.upload_queue.lock().unwrap(); + assert!(upload_queue.queued_operations.len() == 1); } - #[test] - fn redownload_is_not_needed_on_upgrade() { - // originally the implementation missed the `(None, Some(_))` case in the match, and - // proceeded to always redownload if the remote metadata was not available. - - let mut new_sync_tasks = VecDeque::default(); - let sync_id = TenantTimelineId::generate(); - - let local_metadata = dummy_metadata(0x02.into()); - - // type system would in general allow that LayerFileMetadata would be created with - // file_size: None, however `LayerFileMetadata::default` is only allowed from tests, - // and so everywhere within the system valid LayerFileMetadata is being created, it is - // created through `::new`. - let local_files = - HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]); - - let mut remote_entry = RemoteTimeline::new(local_metadata.clone()); - - // RemoteTimeline is constructed out of an older version IndexPart, which didn't carry - // any metadata. - remote_entry - .add_timeline_layers([(PathBuf::from("first_file"), LayerFileMetadata::default())]); - - let (status, sync_needed) = compare_local_and_remote_timeline( - &mut new_sync_tasks, - sync_id, - local_metadata.clone(), - local_files, - &remote_entry, - ); - - assert_eq!( - status, - LocalTimelineInitStatus::LocallyComplete(local_metadata) - ); - assert!(!sync_needed); + // Wait for the uploads to finish + runtime.block_on(client.wait_completion())?; + { + let upload_queue = client.upload_queue.lock().unwrap(); + assert!(upload_queue.queued_operations.is_empty()); + assert!(upload_queue.inprogress_tasks.is_empty()); } - #[test] - fn needs_upload() { - let mut new_sync_tasks = VecDeque::default(); - let sync_id = TenantTimelineId::generate(); - let local_metadata = dummy_metadata(0x02.into()); - let local_files = - HashMap::from([(PathBuf::from("first_file"), LayerFileMetadata::new(123))]); - let mut remote_entry = RemoteTimeline::new(local_metadata.clone()); - remote_entry.add_timeline_layers([]); + // Download back the index.json, and check that the list of files is correct + let index_part = runtime.block_on(client.download_index_file())?; + assert_file_list(&index_part.timeline_layers, &["foo", "bar"]); + let downloaded_metadata = index_part.parse_metadata()?; + assert_eq!(downloaded_metadata, metadata); - let (status, sync_needed) = compare_local_and_remote_timeline( - &mut new_sync_tasks, - sync_id, - local_metadata.clone(), - local_files.clone(), - &remote_entry, - ); - - assert_eq!( - status, - LocalTimelineInitStatus::LocallyComplete(local_metadata.clone()) - ); - assert!(!sync_needed); - - let new_sync_tasks = new_sync_tasks.into_iter().collect::>(); - - assert_eq!( - &new_sync_tasks, - &[( - sync_id, - SyncTask::upload(LayersUpload { - layers_to_upload: local_files, - uploaded_layers: HashMap::default(), - metadata: Some(local_metadata), - }) - )] - ); + // Schedule upload and then a deletion. Check that the deletion is queued + let content_baz = dummy_contents("baz"); + std::fs::write(timeline_path.join("baz"), &content_baz)?; + client.schedule_layer_file_upload( + &timeline_path.join("baz"), + &LayerFileMetadata::new(content_baz.len() as u64), + )?; + client.schedule_layer_file_deletion(&[timeline_path.join("foo")])?; + { + let upload_queue = client.upload_queue.lock().unwrap(); + // Deletion schedules upload of the index file, and the file deletion itself + assert!(upload_queue.queued_operations.len() == 2); + assert!(upload_queue.inprogress_tasks.len() == 1); + assert!(upload_queue.num_inprogress_layer_uploads == 1); + assert!(upload_queue.num_inprogress_deletions == 0); } + assert_remote_files(&["foo", "bar", "index_part.json"], &remote_timeline_dir); + + // Finish them + runtime.block_on(client.wait_completion())?; + + assert_remote_files(&["bar", "baz", "index_part.json"], &remote_timeline_dir); + + Ok(()) } + + // TODO: Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario. + // FIXME: used to have a test for this in upload.rs, `layer_upload_after_local_fs_update()`. + // I didn't understand how it tests that, though. + + // TODO: Test upload failures and retries } diff --git a/pageserver/src/storage_sync/delete.rs b/pageserver/src/storage_sync/delete.rs index 39846f0da3..88567ef624 100644 --- a/pageserver/src/storage_sync/delete.rs +++ b/pageserver/src/storage_sync/delete.rs @@ -1,235 +1,37 @@ -//! Timeline synchronization logic to delete a bulk of timeline's remote files from the remote storage. - -use std::path::Path; - +//! Helper functions to delete files from remote storage with a RemoteStorage use anyhow::Context; -use futures::stream::{FuturesUnordered, StreamExt}; -use tracing::{debug, error, info}; +use std::path::Path; +use tracing::debug; -use crate::storage_sync::{SyncQueue, SyncTask}; use remote_storage::GenericRemoteStorage; -use utils::id::TenantTimelineId; -use super::{LayersDeletion, SyncData}; - -/// Attempts to remove the timleline layers from the remote storage. -/// If the task had not adjusted the metadata before, the deletion will fail. -pub(super) async fn delete_timeline_layers( - storage: &GenericRemoteStorage, - sync_queue: &SyncQueue, - sync_id: TenantTimelineId, - mut delete_data: SyncData, -) -> bool { - if !delete_data.data.deletion_registered { - error!("Cannot delete timeline layers before the deletion metadata is not registered, reenqueueing"); - delete_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Delete(delete_data)); - return false; - } - - if delete_data.data.layers_to_delete.is_empty() { - info!("No layers to delete, skipping"); - return true; - } - - let layers_to_delete = delete_data - .data - .layers_to_delete - .drain() - .collect::>(); - debug!("Layers to delete: {layers_to_delete:?}"); - info!("Deleting {} timeline layers", layers_to_delete.len()); - - let mut delete_tasks = layers_to_delete - .into_iter() - .map(|local_layer_path| async { - match remove_storage_object(storage, &local_layer_path).await { - Ok(()) => Ok(local_layer_path), - Err(e) => Err((e, local_layer_path)), - } - }) - .collect::>(); - - let mut errored = false; - while let Some(deletion_result) = delete_tasks.next().await { - match deletion_result { - Ok(local_layer_path) => { - debug!( - "Successfully deleted layer {} for timeline {sync_id}", - local_layer_path.display() - ); - delete_data.data.deleted_layers.insert(local_layer_path); - } - Err((e, local_layer_path)) => { - errored = true; - error!( - "Failed to delete layer {} for timeline {sync_id}: {e:?}", - local_layer_path.display() - ); - delete_data.data.layers_to_delete.insert(local_layer_path); - } - } - } - - if errored { - debug!("Reenqueuing failed delete task for timeline {sync_id}"); - delete_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Delete(delete_data)); - } else { - info!("Successfully deleted all layers"); - } - errored -} - -async fn remove_storage_object( +pub(super) async fn delete_layer( storage: &GenericRemoteStorage, local_layer_path: &Path, ) -> anyhow::Result<()> { - let storage_path = storage - .remote_object_id(local_layer_path) - .with_context(|| { + async { + debug!( + "Deleting layer from remote storage: {:?}", + local_layer_path.display() + ); + + let storage_path = storage + .remote_object_id(local_layer_path) + .with_context(|| { + format!( + "Failed to get the layer storage path for local path '{}'", + local_layer_path.display() + ) + })?; + + // FIXME: If the deletion fails because the object already didn't exist, + // it would be good to just issue a warning but consider it success. + storage.delete(&storage_path).await.with_context(|| { format!( - "Failed to get the layer storage path for local path '{}'", - local_layer_path.display() + "Failed to delete remote layer from storage at '{:?}'", + storage_path ) - })?; - - storage.delete(&storage_path).await.with_context(|| { - format!( - "Failed to delete remote layer from storage at '{:?}'", - storage_path - ) - }) -} - -#[cfg(test)] -mod tests { - use std::{collections::HashSet, num::NonZeroUsize}; - - use itertools::Itertools; - use tempfile::tempdir; - use tokio::fs; - use utils::lsn::Lsn; - - use crate::{ - storage_sync::test_utils::{create_local_timeline, dummy_metadata}, - tenant::harness::{TenantHarness, TIMELINE_ID}, - }; - use remote_storage::{LocalFs, RemoteStorage}; - - use super::*; - - #[tokio::test] - async fn delete_timeline_negative() -> anyhow::Result<()> { - let harness = TenantHarness::create("delete_timeline_negative")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_path_buf(), - harness.conf.workdir.clone(), - )?); - - let deleted = delete_timeline_layers( - &storage, - &sync_queue, - sync_id, - SyncData { - retries: 1, - data: LayersDeletion { - deleted_layers: HashSet::new(), - layers_to_delete: HashSet::new(), - deletion_registered: false, - }, - }, - ) - .await; - - assert!( - !deleted, - "Should not start the deletion for task with delete metadata unregistered" - ); - - Ok(()) - } - - #[tokio::test] - async fn delete_timeline() -> anyhow::Result<()> { - let harness = TenantHarness::create("delete_timeline")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - let layer_files = ["a", "b", "c", "d"]; - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_path_buf(), - harness.conf.workdir.clone(), - )?); - - let local_storage = storage.as_local().unwrap(); - - let current_retries = 3; - let metadata = dummy_metadata(Lsn(0x30)); - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - let timeline_upload = - create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?; - for (local_path, _metadata) in timeline_upload.layers_to_upload { - let remote_path = - local_storage.resolve_in_storage(&local_storage.remote_object_id(&local_path)?)?; - let remote_parent_dir = remote_path.parent().unwrap(); - if !remote_parent_dir.exists() { - fs::create_dir_all(&remote_parent_dir).await?; - } - fs::copy(&local_path, &remote_path).await?; - } - assert_eq!( - local_storage - .list() - .await? - .into_iter() - .map(|remote_path| local_storage.local_path(&remote_path).unwrap()) - .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) }) - .sorted() - .collect::>(), - layer_files - .iter() - .map(|layer_str| layer_str.to_string()) - .sorted() - .collect::>(), - "Expect to have all layer files remotely before deletion" - ); - - let deleted = delete_timeline_layers( - &storage, - &sync_queue, - sync_id, - SyncData { - retries: current_retries, - data: LayersDeletion { - deleted_layers: HashSet::new(), - layers_to_delete: HashSet::from([ - local_timeline_path.join("a"), - local_timeline_path.join("c"), - local_timeline_path.join("something_different"), - ]), - deletion_registered: true, - }, - }, - ) - .await; - assert!(deleted, "Should be able to delete timeline files"); - - assert_eq!( - local_storage - .list() - .await? - .into_iter() - .map(|remote_path| local_storage.local_path(&remote_path).unwrap()) - .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) }) - .sorted() - .collect::>(), - vec!["b".to_string(), "d".to_string()], - "Expect to have only non-deleted files remotely" - ); - - Ok(()) + }) } + .await } diff --git a/pageserver/src/storage_sync/download.rs b/pageserver/src/storage_sync/download.rs index 6f9b2e2071..59232d4db6 100644 --- a/pageserver/src/storage_sync/download.rs +++ b/pageserver/src/storage_sync/download.rs @@ -1,430 +1,154 @@ -//! Timeline synchronization logic to fetch the layer files from remote storage into pageserver's local directory. +//! Helper functions to download files from remote storage with a RemoteStorage +use std::collections::HashSet; -use std::{ - collections::{HashMap, HashSet}, - fmt::Debug, - mem, - path::Path, -}; - -use anyhow::Context; +use anyhow::{bail, Context}; use futures::stream::{FuturesUnordered, StreamExt}; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use tracing::debug; + +use crate::config::PageServerConf; +use crate::storage_sync::index::LayerFileMetadata; use remote_storage::{DownloadError, GenericRemoteStorage}; -use tokio::{ - fs, - io::{self, AsyncWriteExt}, -}; -use tracing::{debug, error, info, warn}; +use utils::crashsafe::path_with_suffix_extension; +use utils::id::{TenantId, TimelineId}; -use crate::{ - config::PageServerConf, - storage_sync::{index::LayerFileMetadata, SyncTask}, - TEMP_FILE_SUFFIX, -}; -use utils::{ - crashsafe::path_with_suffix_extension, - id::{TenantId, TenantTimelineId, TimelineId}, -}; +use super::index::IndexPart; +use super::RelativePath; -use super::{ - index::{IndexPart, RemoteTimeline}, - LayersDownload, SyncData, SyncQueue, -}; - -// We collect timelines remotely available for each tenant -// in case we failed to gather all index parts (due to an error) -// Poisoned variant is returned. -// When data is received succesfully without errors Present variant is used. -pub enum TenantIndexParts { - Poisoned { - present: HashMap, - missing: HashSet, - }, - Present(HashMap), +async fn fsync_path(path: impl AsRef) -> Result<(), std::io::Error> { + fs::File::open(path).await?.sync_all().await } -impl TenantIndexParts { - fn add_poisoned(&mut self, timeline_id: TimelineId) { - match self { - TenantIndexParts::Poisoned { missing, .. } => { - missing.insert(timeline_id); - } - TenantIndexParts::Present(present) => { - *self = TenantIndexParts::Poisoned { - present: mem::take(present), - missing: HashSet::from([timeline_id]), - } - } - } - } -} - -impl Default for TenantIndexParts { - fn default() -> Self { - TenantIndexParts::Present(HashMap::default()) - } -} - -pub async fn download_index_parts( - conf: &'static PageServerConf, - storage: &GenericRemoteStorage, - keys: HashSet, -) -> HashMap { - let mut index_parts: HashMap = HashMap::new(); - - let mut part_downloads = keys - .into_iter() - .map(|id| async move { (id, download_index_part(conf, storage, id).await) }) - .collect::>(); - - while let Some((id, part_upload_result)) = part_downloads.next().await { - match part_upload_result { - Ok(index_part) => { - debug!("Successfully fetched index part for {id}"); - match index_parts.entry(id.tenant_id).or_default() { - TenantIndexParts::Poisoned { present, .. } => { - present.insert(id.timeline_id, index_part); - } - TenantIndexParts::Present(parts) => { - parts.insert(id.timeline_id, index_part); - } - } - } - Err(download_error) => { - match download_error { - DownloadError::NotFound => { - // thats ok because it means that we didnt upload something we have locally for example - } - e => { - let tenant_parts = index_parts.entry(id.tenant_id).or_default(); - tenant_parts.add_poisoned(id.timeline_id); - error!( - "Failed to fetch index part for {id}: {e} poisoning tenant index parts" - ); - } - } - } - } - } - - index_parts -} - -/// Note: The function is rather expensive from s3 access point of view, it will execute ceil(N/1000) + N requests. -/// At least one request to obtain a list of tenant timelines (more requests is there are more than 1000 timelines). -/// And then will attempt to download all index files that belong to these timelines. -pub async fn gather_tenant_timelines_index_parts( - conf: &'static PageServerConf, - storage: &GenericRemoteStorage, - tenant_id: TenantId, -) -> anyhow::Result> { - let tenant_path = conf.timelines_path(&tenant_id); - let timeline_sync_ids = get_timeline_sync_ids(storage, &tenant_path, tenant_id) - .await - .with_context(|| format!("Failed to list timeline sync ids for tenat {tenant_id}"))?; - - match download_index_parts(conf, storage, timeline_sync_ids) - .await - .remove(&tenant_id) - .ok_or_else(|| anyhow::anyhow!("Missing tenant index parts. This is a bug."))? - { - TenantIndexParts::Poisoned { missing, .. } => { - anyhow::bail!("Failed to download index parts for all timelines. Missing {missing:?}") - } - TenantIndexParts::Present(parts) => Ok(parts), - } -} - -/// Retrieves index data from the remote storage for a given timeline. -async fn download_index_part( - conf: &'static PageServerConf, - storage: &GenericRemoteStorage, - sync_id: TenantTimelineId, -) -> Result { - let index_part_path = conf - .metadata_path(sync_id.timeline_id, sync_id.tenant_id) - .with_file_name(IndexPart::FILE_NAME); - let mut index_part_download = storage - .download_storage_object(None, &index_part_path) - .await?; - - let mut index_part_bytes = Vec::new(); - io::copy( - &mut index_part_download.download_stream, - &mut index_part_bytes, - ) - .await - .with_context(|| { - format!( - "Failed to download an index part into file '{}'", - index_part_path.display() - ) - }) - .map_err(DownloadError::Other)?; - - let index_part: IndexPart = serde_json::from_slice(&index_part_bytes) - .with_context(|| { - format!( - "Failed to deserialize index part file into file '{}'", - index_part_path.display() - ) - }) - .map_err(DownloadError::Other)?; - - let missing_files = index_part.missing_files(); - if !missing_files.is_empty() { - warn!("Found missing layers in index part for timeline {sync_id}: {missing_files:?}"); - } - - Ok(index_part) -} - -/// Timeline download result, with extra data, needed for downloading. -#[derive(Debug)] -pub(super) enum DownloadedTimeline { - /// Remote timeline data is either absent or corrupt, no download possible. - Abort, - /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known. - /// Initial download failed due to some error, the download task is rescheduled for another retry. - FailedAndRescheduled, - /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known. - /// Initial download successful. - Successful(SyncData), -} - -/// Attempts to download all given timeline's layers. -/// Timeline files that already exist locally are skipped during the download, but the local metadata file is -/// updated in the end, if the remote one contains a newer disk_consistent_lsn. /// -/// On an error, bumps the retries count and updates the files to skip with successful downloads, rescheduling the task. -pub(super) async fn download_timeline_layers<'a>( +/// If 'metadata' is given, we will validate that the downloaded file's size matches that +/// in the metadata. (In the future, we might do more cross-checks, like CRC validation) +/// +/// Returns the size of the downloaded file. +pub async fn download_layer_file<'a>( conf: &'static PageServerConf, storage: &'a GenericRemoteStorage, - sync_queue: &'a SyncQueue, - remote_timeline: Option<&'a RemoteTimeline>, - sync_id: TenantTimelineId, - mut download_data: SyncData, -) -> DownloadedTimeline { - let remote_timeline = match remote_timeline { - Some(remote_timeline) => { - if !remote_timeline.awaits_download { - error!("Timeline with sync id {sync_id} is not awaiting download"); - return DownloadedTimeline::Abort; - } - remote_timeline - } - None => { - error!("Timeline with sync id {sync_id} is not present in the remote index"); - return DownloadedTimeline::Abort; - } - }; - - let download = &mut download_data.data; - - let layers_to_download = remote_timeline - .stored_files() - .iter() - .filter_map(|(layer_path, metadata)| { - if !download.layers_to_skip.contains(layer_path) { - Some((layer_path.to_owned(), metadata.to_owned())) - } else { - None - } - }) - .collect::>(); - - debug!("Layers to download: {layers_to_download:?}"); - info!("Downloading {} timeline layers", layers_to_download.len()); - - if layers_to_download.is_empty() { - info!("No layers to download after filtering, skipping"); - return DownloadedTimeline::Successful(download_data); - } - - let mut download_tasks = layers_to_download - .into_iter() - .map(|(layer_destination_path, metadata)| async move { - - match layer_destination_path.metadata() { - Ok(m) if m.is_file() => { - // the file exists from earlier round when we failed after renaming it as - // layer_destination_path - let verified = if let Some(expected) = metadata.file_size() { - m.len() == expected - } else { - // behaviour before recording metadata was to accept any existing - true - }; - - if verified { - debug!( - "Layer already exists locally, skipping download: {}", - layer_destination_path.display() - ); - return Ok((layer_destination_path, LayerFileMetadata::new(m.len()))) - } else { - // no need to remove it, it will be overwritten by fs::rename - // after successful download - warn!("Downloaded layer exists already but layer file metadata mismatches: {}, metadata {:?}", layer_destination_path.display(), metadata); - } - } - Ok(m) => { - return Err(anyhow::anyhow!("Downloaded layer destination exists but is not a file: {m:?}, target needs to be removed/archived manually: {layer_destination_path:?}")); - } - Err(_) => { - // behave as the file didn't exist - } - } - - // Perform a rename inspired by durable_rename from file_utils.c. - // The sequence: - // write(tmp) - // fsync(tmp) - // rename(tmp, new) - // fsync(new) - // fsync(parent) - // For more context about durable_rename check this email from postgres mailing list: - // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com - // If pageserver crashes the temp file will be deleted on startup and re-downloaded. - let temp_file_path = - path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX); - - // TODO: this doesn't use the cached fd for some reason? - let mut destination_file = - fs::File::create(&temp_file_path).await.with_context(|| { - format!( - "Failed to create a destination file for layer '{}'", - temp_file_path.display() - ) - })?; - - let mut layer_download = storage.download_storage_object(None, &layer_destination_path) - .await - .with_context(|| { - format!( - "Failed to initiate the download the layer for {sync_id} into file '{}'", - temp_file_path.display() - ) - })?; - - let bytes_amount = io::copy(&mut layer_download.download_stream, &mut destination_file) - .await - .with_context(|| { - format!( - "Failed to download the layer for {sync_id} into file '{}'", - temp_file_path.display() - ) - })?; - - // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that: - // A file will not be closed immediately when it goes out of scope if there are any IO operations - // that have not yet completed. To ensure that a file is closed immediately when it is dropped, - // you should call flush before dropping it. - // - // From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because - // we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations. - // But for additional safety let's check/wait for any pending operations. - destination_file.flush().await.with_context(|| { - format!( - "failed to flush source file at {}", - temp_file_path.display() - ) - })?; - - match metadata.file_size() { - Some(expected) if expected != bytes_amount => { - anyhow::bail!( - "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'", - temp_file_path.display() - ); - }, - Some(_) | None => { - // matches, or upgrading from an earlier IndexPart version - } - } - - // not using sync_data because it can lose file size update - destination_file.sync_all().await.with_context(|| { - format!( - "failed to fsync source file at {}", - temp_file_path.display() - ) - })?; - drop(destination_file); - - fail::fail_point!("remote-storage-download-pre-rename", |_| { - anyhow::bail!("remote-storage-download-pre-rename failpoint triggered") - }); - - fs::rename(&temp_file_path, &layer_destination_path).await?; - - fsync_path(&layer_destination_path).await.with_context(|| { - format!( - "Cannot fsync layer destination path {}", - layer_destination_path.display(), - ) - })?; - - Ok::<_, anyhow::Error>((layer_destination_path, LayerFileMetadata::new(bytes_amount))) - }) - .collect::>(); - - let mut errors_happened = false; - // keep files we've downloaded to remove them from layers_to_skip if directory fsync fails - let mut undo = HashSet::new(); - while let Some(download_result) = download_tasks.next().await { - match download_result { - Ok((downloaded_path, metadata)) => { - undo.insert(downloaded_path.clone()); - download.layers_to_skip.insert(downloaded_path.clone()); - // what if the key existed already? ignore, because then we would had - // downloaded a partial file, and had to retry - download.gathered_metadata.insert(downloaded_path, metadata); - } - Err(e) => { - errors_happened = true; - error!("Failed to download a layer for timeline {sync_id}: {e:?}"); - } - } - } - - // fsync timeline directory which is a parent directory for downloaded files - let TenantTimelineId { - tenant_id, - timeline_id, - } = &sync_id; - let timeline_dir = conf.timeline_path(timeline_id, tenant_id); - if let Err(e) = fsync_path(&timeline_dir).await { - error!( - "Cannot fsync parent directory {} error {}", - timeline_dir.display(), - e - ); - for item in undo { - download.layers_to_skip.remove(&item); - // intentionally don't clear the gathered_metadata because it exists for fsync_path - // failure on parent directory - } - errors_happened = true; - } - - if errors_happened { - debug!("Reenqueuing failed download task for timeline {sync_id}"); - download_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Download(download_data)); - DownloadedTimeline::FailedAndRescheduled - } else { - info!("Successfully downloaded all layers"); - DownloadedTimeline::Successful(download_data) - } + tenant_id: TenantId, + timeline_id: TimelineId, + path: &'a RelativePath, + layer_metadata: &'a LayerFileMetadata, +) -> anyhow::Result { + download_layer_file_guts(conf, storage, tenant_id, timeline_id, path, layer_metadata).await } -async fn get_timeline_sync_ids( - storage: &GenericRemoteStorage, - tenant_path: &Path, +async fn download_layer_file_guts<'a>( + conf: &'static PageServerConf, + storage: &'a GenericRemoteStorage, tenant_id: TenantId, -) -> anyhow::Result> { - let tenant_storage_path = storage.remote_object_id(tenant_path).with_context(|| { + timeline_id: TimelineId, + path: &'a RelativePath, + layer_metadata: &'a LayerFileMetadata, +) -> anyhow::Result { + const TEMP_DOWNLOAD_EXTENSION: &str = "temp_download"; + + let timeline_path = conf.timeline_path(&timeline_id, &tenant_id); + + let local_path = path.to_local_path(&timeline_path); + + let layer_storage_path = storage.remote_object_id(&local_path).with_context(|| { + format!( + "Failed to get the layer storage path for local path '{}'", + local_path.display() + ) + })?; + + // Perform a rename inspired by durable_rename from file_utils.c. + // The sequence: + // write(tmp) + // fsync(tmp) + // rename(tmp, new) + // fsync(new) + // fsync(parent) + // For more context about durable_rename check this email from postgres mailing list: + // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com + // If pageserver crashes the temp file will be deleted on startup and re-downloaded. + let temp_file_path = path_with_suffix_extension(&local_path, TEMP_DOWNLOAD_EXTENSION); + + // TODO: this doesn't use the cached fd for some reason? + let mut destination_file = fs::File::create(&temp_file_path).await.with_context(|| { + format!( + "Failed to create a destination file for layer '{}'", + temp_file_path.display() + ) + })?; + let mut download = storage + .download(&layer_storage_path) + .await + .with_context(|| { + format!( + "Failed to open a download stream for layer with remote storage path '{layer_storage_path:?}'" + ) + })?; + let bytes_amount = tokio::io::copy(&mut download.download_stream, &mut destination_file).await.with_context(|| { + format!( + "Failed to download layer with remote storage path '{layer_storage_path:?}' into file '{}'", temp_file_path.display() + ) + })?; + + // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that: + // A file will not be closed immediately when it goes out of scope if there are any IO operations + // that have not yet completed. To ensure that a file is closed immediately when it is dropped, + // you should call flush before dropping it. + // + // From the tokio code I see that it waits for pending operations to complete. There shouldt be any because + // we assume that `destination_file` file is fully written. I e there is no pending .write(...).await operations. + // But for additional safety lets check/wait for any pending operations. + destination_file.flush().await.with_context(|| { + format!( + "failed to flush source file at {}", + temp_file_path.display() + ) + })?; + + match layer_metadata.file_size() { + Some(expected) if expected != bytes_amount => { + anyhow::bail!( + "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'", + temp_file_path.display() + ); + } + Some(_) | None => { + // matches, or upgrading from an earlier IndexPart version + } + } + + // not using sync_data because it can lose file size update + destination_file.sync_all().await.with_context(|| { + format!( + "failed to fsync source file at {}", + temp_file_path.display() + ) + })?; + drop(destination_file); + + fail::fail_point!("remote-storage-download-pre-rename", |_| { + bail!("remote-storage-download-pre-rename failpoint triggered") + }); + + fs::rename(&temp_file_path, &local_path).await?; + + fsync_path(&local_path) + .await + .with_context(|| format!("Could not fsync layer file {}", local_path.display(),))?; + + tracing::info!("download complete: {}", local_path.display()); + + Ok(bytes_amount) +} + +/// List timelines of given tenant in remote storage +pub async fn list_remote_timelines<'a>( + storage: &'a GenericRemoteStorage, + conf: &'static PageServerConf, + tenant_id: TenantId, +) -> anyhow::Result> { + let tenant_path = conf.timelines_path(&tenant_id); + let tenant_storage_path = storage.remote_object_id(&tenant_path).with_context(|| { format!( "Failed to get tenant storage path for local path '{}'", tenant_path.display() @@ -444,7 +168,8 @@ async fn get_timeline_sync_ids( anyhow::bail!("no timelines found on the remote storage") } - let mut sync_ids = HashSet::new(); + let mut timeline_ids = HashSet::new(); + let mut part_downloads = FuturesUnordered::new(); for timeline_remote_storage_key in timelines { let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| { @@ -455,241 +180,78 @@ async fn get_timeline_sync_ids( format!("failed to parse object name into timeline id '{object_name}'") })?; - sync_ids.insert(TenantTimelineId { - tenant_id, - timeline_id, - }); + // list_prefixes returns all files with the prefix. If we haven't seen this timeline ID + // yet, launch a download task for it. + if !timeline_ids.contains(&timeline_id) { + timeline_ids.insert(timeline_id); + let storage_clone = storage.clone(); + part_downloads.push(async move { + ( + timeline_id, + download_index_part(conf, &storage_clone, tenant_id, timeline_id).await, + ) + }); + } } - Ok(sync_ids) + // Wait for all the download tasks to complete. + let mut timeline_parts = Vec::new(); + while let Some((timeline_id, part_upload_result)) = part_downloads.next().await { + let index_part = part_upload_result + .with_context(|| format!("Failed to fetch index part for timeline {timeline_id}"))?; + + debug!("Successfully fetched index part for timeline {timeline_id}"); + timeline_parts.push((timeline_id, index_part)); + } + Ok(timeline_parts) } -async fn fsync_path(path: impl AsRef) -> Result<(), io::Error> { - fs::File::open(path).await?.sync_all().await -} +pub async fn download_index_part( + conf: &'static PageServerConf, + storage: &GenericRemoteStorage, + tenant_id: TenantId, + timeline_id: TimelineId, +) -> Result { + async { + let index_part_path = conf + .metadata_path(timeline_id, tenant_id) + .with_file_name(IndexPart::FILE_NAME); + let part_storage_path = storage + .remote_object_id(&index_part_path) + .with_context(|| { + format!( + "Failed to get the index part storage path for local path '{}'", + index_part_path.display() + ) + }) + .map_err(DownloadError::BadInput)?; -#[cfg(test)] -mod tests { - use std::{ - collections::{BTreeSet, HashSet}, - num::NonZeroUsize, - path::PathBuf, - }; + let mut index_part_download = storage.download(&part_storage_path).await?; - use remote_storage::{LocalFs, RemoteStorage}; - use tempfile::tempdir; - use utils::lsn::Lsn; - - use crate::{ - storage_sync::{ - index::RelativePath, - test_utils::{create_local_timeline, dummy_metadata}, - }, - tenant::harness::{TenantHarness, TIMELINE_ID}, - }; - - use super::*; - - #[tokio::test] - async fn download_timeline() -> anyhow::Result<()> { - let harness = TenantHarness::create("download_timeline")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - let layer_files = ["a", "b", "layer_to_skip", "layer_to_keep_locally"]; - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_owned(), - harness.conf.workdir.clone(), - )?); - let local_storage = storage.as_local().unwrap(); - let current_retries = 3; - let metadata = dummy_metadata(Lsn(0x30)); - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - let timeline_upload = - create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?; - - for local_path in timeline_upload.layers_to_upload.keys() { - let remote_path = - local_storage.resolve_in_storage(&storage.remote_object_id(local_path)?)?; - let remote_parent_dir = remote_path.parent().unwrap(); - if !remote_parent_dir.exists() { - fs::create_dir_all(&remote_parent_dir).await?; - } - fs::copy(&local_path, &remote_path).await?; - } - let mut read_dir = fs::read_dir(&local_timeline_path).await?; - while let Some(dir_entry) = read_dir.next_entry().await? { - if dir_entry.file_name().to_str() == Some("layer_to_keep_locally") { - continue; - } else { - fs::remove_file(dir_entry.path()).await?; - } - } - - let mut remote_timeline = RemoteTimeline::new(metadata.clone()); - remote_timeline.awaits_download = true; - remote_timeline.add_timeline_layers(layer_files.iter().map(|layer| { - let layer_path = local_timeline_path.join(layer); - - // this could had also been LayerFileMetadata::default(), but since in this test we - // don't do the merge operation done by storage_sync::download_timeline_data, it would - // not be merged back to timeline. - let metadata_from_upload = timeline_upload - .layers_to_upload - .get(&layer_path) - .expect("layer must exist in previously uploaded paths") - .to_owned(); - (layer_path, metadata_from_upload) - })); - - let download_data = match download_timeline_layers( - harness.conf, - &storage, - &sync_queue, - Some(&remote_timeline), - sync_id, - SyncData::new( - current_retries, - LayersDownload::from_skipped_layers(HashSet::from([ - local_timeline_path.join("layer_to_skip") - ])), - ), + let mut index_part_bytes = Vec::new(); + tokio::io::copy( + &mut index_part_download.download_stream, + &mut index_part_bytes, ) .await - { - DownloadedTimeline::Successful(data) => data, - wrong_result => { - panic!("Expected a successful download for timeline, but got: {wrong_result:?}") - } - }; + .with_context(|| { + format!( + "Failed to download an index part into file '{}'", + index_part_path.display() + ) + }) + .map_err(DownloadError::Other)?; - assert_eq!( - current_retries, download_data.retries, - "On successful download, retries are not expected to change" - ); - assert_eq!( - download_data - .data - .layers_to_skip - .into_iter() - .collect::>(), - layer_files - .iter() - .map(|layer| local_timeline_path.join(layer)) - .collect(), - "On successful download, layers to skip should contain all downloaded files and present layers that were skipped" - ); + let index_part: IndexPart = serde_json::from_slice(&index_part_bytes) + .with_context(|| { + format!( + "Failed to deserialize index part file into file '{}'", + index_part_path.display() + ) + }) + .map_err(DownloadError::Other)?; - let mut downloaded_files = BTreeSet::new(); - let mut read_dir = fs::read_dir(&local_timeline_path).await?; - while let Some(dir_entry) = read_dir.next_entry().await? { - downloaded_files.insert(dir_entry.path()); - } - - assert_eq!( - downloaded_files, - layer_files - .iter() - .filter(|layer| layer != &&"layer_to_skip") - .map(|layer| local_timeline_path.join(layer)) - .collect(), - "On successful download, all layers that were not skipped, should be downloaded" - ); - - Ok(()) - } - - #[tokio::test] - async fn download_timeline_negatives() -> anyhow::Result<()> { - let harness = TenantHarness::create("download_timeline_negatives")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_owned(), - harness.conf.workdir.clone(), - )?); - - let empty_remote_timeline_download = download_timeline_layers( - harness.conf, - &storage, - &sync_queue, - None, - sync_id, - SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())), - ) - .await; - assert!( - matches!(empty_remote_timeline_download, DownloadedTimeline::Abort), - "Should not allow downloading for empty remote timeline" - ); - - let not_expecting_download_remote_timeline = RemoteTimeline::new(dummy_metadata(Lsn(5))); - assert!( - !not_expecting_download_remote_timeline.awaits_download, - "Should not expect download for the timeline" - ); - let already_downloading_remote_timeline_download = download_timeline_layers( - harness.conf, - &storage, - &sync_queue, - Some(¬_expecting_download_remote_timeline), - sync_id, - SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())), - ) - .await; - assert!( - matches!( - already_downloading_remote_timeline_download, - DownloadedTimeline::Abort, - ), - "Should not allow downloading for remote timeline that does not expect it" - ); - - Ok(()) - } - - #[tokio::test] - async fn test_download_index_part() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_download_index_part")?; - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_owned(), - harness.conf.workdir.clone(), - )?); - let local_storage = storage.as_local().unwrap(); - let metadata = dummy_metadata(Lsn(0x30)); - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - - let index_part = IndexPart::new( - HashSet::from([ - RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?, - RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?, - ]), - HashSet::from([RelativePath::new( - &local_timeline_path, - local_timeline_path.join("three"), - )?]), - metadata.disk_consistent_lsn(), - metadata.to_bytes()?, - ); - - let local_index_part_path = harness - .conf - .metadata_path(sync_id.timeline_id, sync_id.tenant_id) - .with_file_name(IndexPart::FILE_NAME); - let index_part_remote_id = local_storage.remote_object_id(&local_index_part_path)?; - let index_part_local_path = PathBuf::from(index_part_remote_id.to_string()); - fs::create_dir_all(index_part_local_path.parent().unwrap()).await?; - fs::write(&index_part_local_path, serde_json::to_vec(&index_part)?).await?; - - let downloaded_index_part = download_index_part(harness.conf, &storage, sync_id).await?; - - assert_eq!( - downloaded_index_part, index_part, - "Downloaded index part should be the same as the one in storage" - ); - - Ok(()) + Ok(index_part) } + .await } diff --git a/pageserver/src/storage_sync/index.rs b/pageserver/src/storage_sync/index.rs index 0779d95e8e..b1f43dcb93 100644 --- a/pageserver/src/storage_sync/index.rs +++ b/pageserver/src/storage_sync/index.rs @@ -2,26 +2,18 @@ //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about //! remote timeline layers and its metadata. -use std::ops::{Deref, DerefMut}; use std::{ collections::{HashMap, HashSet}, path::{Path, PathBuf}, - sync::Arc, }; -use anyhow::{anyhow, Context, Ok}; +use anyhow::{Context, Ok}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; -use tokio::sync::RwLock; -use tracing::log::warn; -use crate::{config::PageServerConf, tenant::metadata::TimelineMetadata}; -use utils::{ - id::{TenantId, TenantTimelineId, TimelineId}, - lsn::Lsn, -}; +use crate::tenant::metadata::TimelineMetadata; -use super::download::TenantIndexParts; +use utils::lsn::Lsn; /// A part of the filesystem path, that needs a root to become a path again. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] @@ -30,270 +22,23 @@ pub struct RelativePath(String); impl RelativePath { /// Attempts to strip off the base from path, producing a relative path or an error. - pub fn new>(base: &Path, path: P) -> anyhow::Result { - let path = path.as_ref(); - let relative = path.strip_prefix(base).with_context(|| { + pub fn from_local_path(timeline_path: &Path, path: &Path) -> anyhow::Result { + let relative = path.strip_prefix(timeline_path).with_context(|| { format!( "path '{}' is not relative to base '{}'", path.display(), - base.display() + timeline_path.display() ) })?; - Ok(RelativePath(relative.to_string_lossy().to_string())) + Ok(Self::from_filename(relative)) } - /// Joins the relative path with the base path. - fn as_path(&self, base: &Path) -> PathBuf { - base.join(&self.0) - } -} - -#[derive(Debug, Clone, Default)] -pub struct TenantEntry(HashMap); - -impl TenantEntry { - pub fn has_in_progress_downloads(&self) -> bool { - self.values() - .any(|remote_timeline| remote_timeline.awaits_download) - } -} - -impl Deref for TenantEntry { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for TenantEntry { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl From> for TenantEntry { - fn from(inner: HashMap) -> Self { - Self(inner) - } -} - -/// An index to track tenant files that exist on the remote storage. -#[derive(Debug, Clone, Default)] -pub struct RemoteTimelineIndex { - entries: HashMap, -} - -/// A wrapper to synchronize the access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`]. -#[derive(Default)] -pub struct RemoteIndex(Arc>); - -impl RemoteIndex { - pub fn from_parts( - conf: &'static PageServerConf, - index_parts: HashMap, - ) -> anyhow::Result { - let mut entries: HashMap = HashMap::new(); - - for (tenant_id, index_parts) in index_parts { - match index_parts { - // TODO: should we schedule a retry so it can be recovered? otherwise we can revive it only through detach/attach or pageserver restart - TenantIndexParts::Poisoned { missing, ..} => warn!("skipping tenant_id set up for remote index because the index download has failed for timeline(s): {missing:?}"), - TenantIndexParts::Present(timelines) => { - for (timeline_id, index_part) in timelines { - let timeline_path = conf.timeline_path(&timeline_id, &tenant_id); - let remote_timeline = - RemoteTimeline::from_index_part(&timeline_path, index_part) - .context("Failed to restore remote timeline data from index part")?; - - entries - .entry(tenant_id) - .or_default() - .insert(timeline_id, remote_timeline); - } - }, - } - } - - Ok(Self(Arc::new(RwLock::new(RemoteTimelineIndex { entries })))) + pub fn from_filename(path: &Path) -> RelativePath { + RelativePath(path.to_string_lossy().to_string()) } - pub async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, RemoteTimelineIndex> { - self.0.read().await - } - - pub async fn write(&self) -> tokio::sync::RwLockWriteGuard<'_, RemoteTimelineIndex> { - self.0.write().await - } -} - -impl Clone for RemoteIndex { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } -} - -impl RemoteTimelineIndex { - pub fn timeline_entry( - &self, - TenantTimelineId { - tenant_id, - timeline_id, - }: &TenantTimelineId, - ) -> Option<&RemoteTimeline> { - self.entries.get(tenant_id)?.get(timeline_id) - } - - pub fn timeline_entry_mut( - &mut self, - TenantTimelineId { - tenant_id, - timeline_id, - }: &TenantTimelineId, - ) -> Option<&mut RemoteTimeline> { - self.entries.get_mut(tenant_id)?.get_mut(timeline_id) - } - - pub fn add_timeline_entry( - &mut self, - TenantTimelineId { - tenant_id, - timeline_id, - }: TenantTimelineId, - entry: RemoteTimeline, - ) { - self.entries - .entry(tenant_id) - .or_default() - .insert(timeline_id, entry); - } - - pub fn remove_timeline_entry( - &mut self, - TenantTimelineId { - tenant_id, - timeline_id, - }: TenantTimelineId, - ) -> Option { - self.entries - .entry(tenant_id) - .or_default() - .remove(&timeline_id) - } - - pub fn tenant_entry(&self, tenant_id: &TenantId) -> Option<&TenantEntry> { - self.entries.get(tenant_id) - } - - pub fn tenant_entry_mut(&mut self, tenant_id: &TenantId) -> Option<&mut TenantEntry> { - self.entries.get_mut(tenant_id) - } - - pub fn add_tenant_entry(&mut self, tenant_id: TenantId) -> &mut TenantEntry { - self.entries.entry(tenant_id).or_default() - } - - pub fn remove_tenant_entry(&mut self, tenant_id: &TenantId) -> Option { - self.entries.remove(tenant_id) - } - - pub fn set_awaits_download( - &mut self, - id: &TenantTimelineId, - awaits_download: bool, - ) -> anyhow::Result<()> { - self.timeline_entry_mut(id) - .ok_or_else(|| anyhow!("unknown timeline sync {id}"))? - .awaits_download = awaits_download; - Ok(()) - } -} - -/// Restored index part data about the timeline, stored in the remote index. -#[derive(Debug, Clone)] -pub struct RemoteTimeline { - timeline_layers: HashMap, - missing_layers: HashMap, - - pub metadata: TimelineMetadata, - pub awaits_download: bool, -} - -impl RemoteTimeline { - pub fn new(metadata: TimelineMetadata) -> Self { - Self { - timeline_layers: HashMap::default(), - missing_layers: HashMap::default(), - metadata, - awaits_download: false, - } - } - - pub fn add_timeline_layers( - &mut self, - new_layers: impl IntoIterator, - ) { - self.timeline_layers.extend(new_layers); - } - - pub fn add_upload_failures( - &mut self, - upload_failures: impl IntoIterator, - ) { - self.missing_layers.extend(upload_failures); - } - - pub fn remove_layers(&mut self, layers_to_remove: &HashSet) { - self.timeline_layers - .retain(|layer, _| !layers_to_remove.contains(layer)); - self.missing_layers - .retain(|layer, _| !layers_to_remove.contains(layer)); - } - - /// Lists all layer files in the given remote timeline. Omits the metadata file. - pub fn stored_files(&self) -> &HashMap { - &self.timeline_layers - } - - /// Combines metadata gathered or verified during downloading needed layer files to metadata on - /// the [`RemoteIndex`], so it can be uploaded later. - pub fn merge_metadata_from_downloaded( - &mut self, - downloaded: &HashMap, - ) { - downloaded.iter().for_each(|(path, metadata)| { - if let Some(upgraded) = self.timeline_layers.get_mut(path) { - upgraded.merge(metadata); - } - }); - } - - pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result { - let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?; - let default_metadata = &IndexLayerMetadata::default(); - - let find_metadata = |key: &RelativePath| -> LayerFileMetadata { - index_part - .layer_metadata - .get(key) - .unwrap_or(default_metadata) - .into() - }; - - Ok(Self { - timeline_layers: index_part - .timeline_layers - .iter() - .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path))) - .collect(), - missing_layers: index_part - .missing_layers - .iter() - .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path))) - .collect(), - metadata, - awaits_download: false, - }) + pub fn to_local_path(&self, timeline_path: &Path) -> PathBuf { + timeline_path.join(&self.0) } } @@ -322,6 +67,10 @@ impl LayerFileMetadata { } } + /// This is used to initialize the metadata for remote layers, for which + /// the metadata was missing from the index part file. + pub const MISSING: Self = LayerFileMetadata { file_size: None }; + pub fn file_size(&self) -> Option { self.file_size } @@ -335,7 +84,8 @@ impl LayerFileMetadata { } } -/// Part of the remote index, corresponding to a certain timeline. +/// In-memory representation of an `index_part.json` file +/// /// Contains the data about all files in the timeline, present remotely and its metadata. /// /// This type needs to be backwards and forwards compatible. When changing the fields, @@ -350,13 +100,10 @@ pub struct IndexPart { /// Each of the layers present on remote storage. /// /// Additional metadata can might exist in `layer_metadata`. - timeline_layers: HashSet, + pub timeline_layers: HashSet, - /// Currently is not really used in pageserver, - /// present to manually keep track of the layer files that pageserver might never retrieve. - /// - /// Such "holes" might appear if any upload task was evicted on an error threshold: - /// the this layer will only be rescheduled for upload on pageserver restart. + /// FIXME: unused field. This should be removed, but that changes the on-disk format, + /// so we need to make sure we're backwards- (and maybe forwards-) compatible missing_layers: HashSet, /// Per layer file metadata, which can be present for a present or missing layer file. @@ -364,10 +111,12 @@ pub struct IndexPart { /// Older versions of `IndexPart` will not have this property or have only a part of metadata /// that latest version stores. #[serde(default)] - layer_metadata: HashMap, + pub layer_metadata: HashMap, + // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata. + // It's duplicated here for convenience. #[serde_as(as = "DisplayFromStr")] - disk_consistent_lsn: Lsn, + pub disk_consistent_lsn: Lsn, metadata_bytes: Vec, } @@ -379,63 +128,32 @@ impl IndexPart { const LATEST_VERSION: usize = 1; pub const FILE_NAME: &'static str = "index_part.json"; - #[cfg(test)] pub fn new( - timeline_layers: HashSet, - missing_layers: HashSet, + layers_and_metadata: HashMap, disk_consistent_lsn: Lsn, metadata_bytes: Vec, ) -> Self { + let mut timeline_layers = HashSet::new(); + let mut layer_metadata = HashMap::new(); + + separate_paths_and_metadata( + &layers_and_metadata, + &mut timeline_layers, + &mut layer_metadata, + ); + Self { version: Self::LATEST_VERSION, timeline_layers, - missing_layers, - layer_metadata: HashMap::default(), + missing_layers: HashSet::new(), + layer_metadata, disk_consistent_lsn, metadata_bytes, } } - pub fn missing_files(&self) -> &HashSet { - &self.missing_layers - } - - pub fn from_remote_timeline( - timeline_path: &Path, - remote_timeline: RemoteTimeline, - ) -> anyhow::Result { - let metadata_bytes = remote_timeline.metadata.to_bytes()?; - - let mut layer_metadata = HashMap::new(); - - let mut missing_layers = HashSet::new(); - - separate_paths_and_metadata( - timeline_path, - &remote_timeline.missing_layers, - &mut missing_layers, - &mut layer_metadata, - ) - .context("Failed to convert missing layers' paths to relative ones")?; - - let mut timeline_layers = HashSet::new(); - - separate_paths_and_metadata( - timeline_path, - &remote_timeline.timeline_layers, - &mut timeline_layers, - &mut layer_metadata, - ) - .context("Failed to convert timeline layers' paths to relative ones")?; - - Ok(Self { - version: Self::LATEST_VERSION, - timeline_layers, - missing_layers, - layer_metadata, - disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(), - metadata_bytes, - }) + pub fn parse_metadata(&self) -> anyhow::Result { + TimelineMetadata::from_bytes(&self.metadata_bytes) } } @@ -454,202 +172,20 @@ impl From<&'_ LayerFileMetadata> for IndexLayerMetadata { } fn separate_paths_and_metadata( - timeline_path: &Path, - input: &HashMap, + input: &HashMap, output: &mut HashSet, layer_metadata: &mut HashMap, -) -> anyhow::Result<()> { +) { for (path, metadata) in input { - let rel_path = RelativePath::new(timeline_path, path)?; let metadata = IndexLayerMetadata::from(metadata); - - layer_metadata.insert(rel_path.clone(), metadata); - output.insert(rel_path); + layer_metadata.insert(path.clone(), metadata); + output.insert(path.clone()); } - Ok(()) } #[cfg(test)] mod tests { - use std::collections::BTreeSet; - use super::*; - use crate::tenant::harness::{TenantHarness, TIMELINE_ID}; - use crate::DEFAULT_PG_VERSION; - - #[test] - fn index_part_conversion() { - let harness = TenantHarness::create("index_part_conversion").unwrap(); - let timeline_path = harness.timeline_path(&TIMELINE_ID); - let metadata = TimelineMetadata::new( - Lsn(5).align(), - Some(Lsn(4)), - None, - Lsn(3), - Lsn(2), - Lsn(1), - DEFAULT_PG_VERSION, - ); - let remote_timeline = RemoteTimeline { - timeline_layers: HashMap::from([ - (timeline_path.join("layer_1"), LayerFileMetadata::new(1)), - (timeline_path.join("layer_2"), LayerFileMetadata::new(2)), - ]), - missing_layers: HashMap::from([ - (timeline_path.join("missing_1"), LayerFileMetadata::new(3)), - (timeline_path.join("missing_2"), LayerFileMetadata::new(4)), - ]), - metadata: metadata.clone(), - awaits_download: false, - }; - - let index_part = IndexPart::from_remote_timeline(&timeline_path, remote_timeline.clone()) - .expect("Correct remote timeline should be convertible to index part"); - - assert_eq!( - index_part.timeline_layers.iter().collect::>(), - BTreeSet::from([ - &RelativePath("layer_1".to_string()), - &RelativePath("layer_2".to_string()) - ]), - "Index part should have all remote timeline layers after the conversion" - ); - assert_eq!( - index_part.missing_layers.iter().collect::>(), - BTreeSet::from([ - &RelativePath("missing_1".to_string()), - &RelativePath("missing_2".to_string()) - ]), - "Index part should have all missing remote timeline layers after the conversion" - ); - assert_eq!( - index_part.disk_consistent_lsn, - metadata.disk_consistent_lsn(), - "Index part should have disk consistent lsn from the timeline" - ); - assert_eq!( - index_part.metadata_bytes, - metadata - .to_bytes() - .expect("Failed to serialize correct metadata into bytes"), - "Index part should have all missing remote timeline layers after the conversion" - ); - - let restored_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part) - .expect("Correct index part should be convertible to remote timeline"); - - let original_metadata = &remote_timeline.metadata; - let restored_metadata = &restored_timeline.metadata; - // we have to compare the metadata this way, since its header is different after creation and restoration, - // but that is now consireded ok. - assert_eq!( - original_metadata.disk_consistent_lsn(), - restored_metadata.disk_consistent_lsn(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - assert_eq!( - original_metadata.prev_record_lsn(), - restored_metadata.prev_record_lsn(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - assert_eq!( - original_metadata.ancestor_timeline(), - restored_metadata.ancestor_timeline(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - assert_eq!( - original_metadata.ancestor_lsn(), - restored_metadata.ancestor_lsn(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - assert_eq!( - original_metadata.latest_gc_cutoff_lsn(), - restored_metadata.latest_gc_cutoff_lsn(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - assert_eq!( - original_metadata.initdb_lsn(), - restored_metadata.initdb_lsn(), - "remote timeline -> index part -> remote timeline conversion should not alter metadata" - ); - - assert_eq!( - remote_timeline.awaits_download, restored_timeline.awaits_download, - "remote timeline -> index part -> remote timeline conversion should not loose download flag" - ); - - assert_eq!( - remote_timeline - .timeline_layers - .into_iter() - .collect::>(), - restored_timeline - .timeline_layers - .into_iter() - .collect::>(), - "remote timeline -> index part -> remote timeline conversion should not loose layer data" - ); - assert_eq!( - remote_timeline - .missing_layers - .into_iter() - .collect::>(), - restored_timeline - .missing_layers - .into_iter() - .collect::>(), - "remote timeline -> index part -> remote timeline conversion should not loose missing file data" - ); - } - - #[test] - fn index_part_conversion_negatives() { - let harness = TenantHarness::create("index_part_conversion_negatives").unwrap(); - let timeline_path = harness.timeline_path(&TIMELINE_ID); - let metadata = TimelineMetadata::new( - Lsn(5).align(), - Some(Lsn(4)), - None, - Lsn(3), - Lsn(2), - Lsn(1), - DEFAULT_PG_VERSION, - ); - - let conversion_result = IndexPart::from_remote_timeline( - &timeline_path, - RemoteTimeline { - timeline_layers: HashMap::from([ - (PathBuf::from("bad_path"), LayerFileMetadata::new(1)), - (timeline_path.join("layer_2"), LayerFileMetadata::new(2)), - ]), - missing_layers: HashMap::from([ - (timeline_path.join("missing_1"), LayerFileMetadata::new(3)), - (timeline_path.join("missing_2"), LayerFileMetadata::new(4)), - ]), - metadata: metadata.clone(), - awaits_download: false, - }, - ); - assert!(conversion_result.is_err(), "Should not be able to convert metadata with layer paths that are not in the timeline directory"); - - let conversion_result = IndexPart::from_remote_timeline( - &timeline_path, - RemoteTimeline { - timeline_layers: HashMap::from([ - (timeline_path.join("layer_1"), LayerFileMetadata::new(1)), - (timeline_path.join("layer_2"), LayerFileMetadata::new(2)), - ]), - missing_layers: HashMap::from([ - (PathBuf::from("bad_path"), LayerFileMetadata::new(3)), - (timeline_path.join("missing_2"), LayerFileMetadata::new(4)), - ]), - metadata, - awaits_download: false, - }, - ); - assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory"); - } #[test] fn v0_indexpart_is_parsed() { diff --git a/pageserver/src/storage_sync/upload.rs b/pageserver/src/storage_sync/upload.rs index f91105052b..d3fd797b53 100644 --- a/pageserver/src/storage_sync/upload.rs +++ b/pageserver/src/storage_sync/upload.rs @@ -1,479 +1,110 @@ -//! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints. +//! Helper functions to upload files to remote storage with a RemoteStorage -use std::{fmt::Debug, path::PathBuf}; - -use anyhow::Context; -use futures::stream::{FuturesUnordered, StreamExt}; -use remote_storage::GenericRemoteStorage; +use anyhow::{bail, Context}; +use fail::fail_point; +use std::path::Path; use tokio::fs; -use tracing::{debug, error, info, warn}; -use utils::id::TenantTimelineId; - -use super::{ - index::{IndexPart, RemoteTimeline}, - LayersUpload, SyncData, SyncQueue, -}; -use crate::metrics::NO_LAYERS_UPLOAD; -use crate::{config::PageServerConf, storage_sync::SyncTask}; +use super::index::IndexPart; +use crate::config::PageServerConf; +use crate::storage_sync::LayerFileMetadata; +use remote_storage::GenericRemoteStorage; +use utils::id::{TenantId, TimelineId}; /// Serializes and uploads the given index part data to the remote storage. -pub(super) async fn upload_index_part( +pub(super) async fn upload_index_part<'a>( conf: &'static PageServerConf, - storage: &GenericRemoteStorage, - sync_id: TenantTimelineId, - index_part: IndexPart, + storage: &'a GenericRemoteStorage, + tenant_id: TenantId, + timeline_id: TimelineId, + index_part: &'a IndexPart, ) -> anyhow::Result<()> { - let index_part_bytes = serde_json::to_vec(&index_part) - .context("Failed to serialize index part file into bytes")?; - let index_part_size = index_part_bytes.len(); - let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes)); + fail_point!("before-upload-index", |_| { + bail!("failpoint before-upload-index") + }); + async { + let index_part_bytes = serde_json::to_vec(&index_part) + .context("Failed to serialize index part file into bytes")?; + let index_part_size = index_part_bytes.len(); + let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes)); - let index_part_path = conf - .metadata_path(sync_id.timeline_id, sync_id.tenant_id) - .with_file_name(IndexPart::FILE_NAME); - storage - .upload_storage_object( - Box::new(index_part_bytes), - index_part_size, - &index_part_path, - ) - .await - .with_context(|| format!("Failed to upload index part for '{sync_id}'")) -} - -/// Timeline upload result, with extra data, needed for uploading. -#[derive(Debug)] -pub(super) enum UploadedTimeline { - /// Upload failed due to some error, the upload task is rescheduled for another retry. - FailedAndRescheduled(anyhow::Error), - /// No issues happened during the upload, all task files were put into the remote storage. - Successful(SyncData), + let index_part_path = conf + .metadata_path(timeline_id, tenant_id) + .with_file_name(IndexPart::FILE_NAME); + storage + .upload_storage_object( + Box::new(index_part_bytes), + index_part_size, + &index_part_path, + ) + .await + .with_context(|| { + format!("Failed to upload index part for '{tenant_id} / {timeline_id}'") + }) + } + .await } /// Attempts to upload given layer files. /// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload. /// /// On an error, bumps the retries count and reschedules the entire task. -pub(super) async fn upload_timeline_layers<'a>( - storage: &'a GenericRemoteStorage, - sync_queue: &SyncQueue, - remote_timeline: Option<&'a RemoteTimeline>, - sync_id: TenantTimelineId, - mut upload_data: SyncData, -) -> UploadedTimeline { - let upload = &mut upload_data.data; - let new_upload_lsn = upload - .metadata - .as_ref() - .map(|meta| meta.disk_consistent_lsn()); +pub(super) async fn upload_timeline_layer( + storage: &GenericRemoteStorage, + source_path: &Path, + known_metadata: &LayerFileMetadata, +) -> anyhow::Result<()> { + fail_point!("before-upload-layer", |_| { + bail!("failpoint before-upload-layer") + }); + async { + let storage_path = storage.remote_object_id(source_path).with_context(|| { + format!( + "Failed to get the layer storage path for local path '{}'", + source_path.display() + ) + })?; - let already_uploaded_layers = remote_timeline - .map(|timeline| { - timeline - .stored_files() - .keys() - .cloned() - .collect::>() - }) - .unwrap_or_default(); + let source_file = fs::File::open(&source_path).await.with_context(|| { + format!( + "Failed to open a source file for layer '{}'", + source_path.display() + ) + })?; - let layers_to_upload = upload - .layers_to_upload - .iter() - .filter_map(|(k, v)| { - if !already_uploaded_layers.contains(k) { - Some((k.to_owned(), v.to_owned())) - } else { - None - } - }) - .collect::>(); - - if layers_to_upload.is_empty() { - debug!("No layers to upload after filtering, aborting"); - NO_LAYERS_UPLOAD - .with_label_values(&[ - &sync_id.tenant_id.to_string(), - &sync_id.timeline_id.to_string(), - ]) - .inc(); - return UploadedTimeline::Successful(upload_data); - } - - debug!("Layers to upload: {layers_to_upload:?}"); - info!( - "Uploading {} timeline layers, new lsn: {new_upload_lsn:?}", - layers_to_upload.len(), - ); - - let mut upload_tasks = layers_to_upload - .into_iter() - .map(|(source_path, known_metadata)| async move { - let source_file = match fs::File::open(&source_path).await.with_context(|| { + let fs_size = source_file + .metadata() + .await + .with_context(|| { format!( - "Failed to upen a source file for layer '{}'", + "Failed to get the source file metadata for layer '{}'", source_path.display() ) - }) { - Ok(file) => file, - Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)), - }; + })? + .len(); - let fs_size = source_file - .metadata() - .await - .with_context(|| { - format!( - "Failed to get the source file metadata for layer '{}'", - source_path.display() - ) - }) - .map_err(UploadError::Other)? - .len(); - - // FIXME: this looks bad - if let Some(metadata_size) = known_metadata.file_size() { - if metadata_size != fs_size { - return Err(UploadError::Other(anyhow::anyhow!( - "File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}" - ))); - } - } else { - // this is a silly state we would like to avoid + // FIXME: this looks bad + if let Some(metadata_size) = known_metadata.file_size() { + if metadata_size != fs_size { + bail!("File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"); } - - let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize")) - .map_err(UploadError::Other)?; - - match storage - .upload_storage_object(Box::new(source_file), fs_size, &source_path) - .await - .with_context(|| format!("Failed to upload layer file for {sync_id}")) - { - Ok(()) => Ok(source_path), - Err(e) => Err(UploadError::MissingLocalFile(source_path, e)), - } - }) - .collect::>(); - - let mut errors = Vec::new(); - while let Some(upload_result) = upload_tasks.next().await { - match upload_result { - Ok(uploaded_path) => { - let metadata = upload - .layers_to_upload - .remove(&uploaded_path) - .expect("metadata should always exist, assuming no double uploads"); - upload.uploaded_layers.insert(uploaded_path, metadata); - } - Err(e) => match e { - UploadError::Other(e) => { - error!("Failed to upload a layer for timeline {sync_id}: {e:?}"); - errors.push(format!("{e:#}")); - } - UploadError::MissingLocalFile(source_path, e) => { - if source_path.exists() { - error!("Failed to upload a layer for timeline {sync_id}: {e:?}"); - errors.push(format!("{e:#}")); - } else { - // We have run the upload sync task, but the file we wanted to upload is gone. - // This is "fine" due the asynchronous nature of the sync loop: it only reacts to events and might need to - // retry the upload tasks, if S3 or network is down: but during this time, pageserver might still operate and - // run compaction/gc tasks, removing redundant files from disk. - // It's not good to pause GC/compaction because of those and we would rather skip such uploads. - // - // Yet absence of such files might also mean that the timeline metadata file was updated (GC moves the Lsn forward, for instance). - // We don't try to read a more recent version, since it could contain `disk_consistent_lsn` that does not have its upload finished yet. - // This will create "missing" layers and make data inconsistent. - // Instead, we only update the metadata when it was submitted in an upload task as a checkpoint result. - upload.layers_to_upload.remove(&source_path); - warn!( - "Missing locally a layer file {} scheduled for upload, skipping", - source_path.display() - ); - } - } - }, + } else { + // this is a silly state we would like to avoid } - } - if errors.is_empty() { - info!("Successfully uploaded all layers"); - UploadedTimeline::Successful(upload_data) - } else { - debug!("Reenqueuing failed upload task for timeline {sync_id}"); - upload_data.retries += 1; - sync_queue.push(sync_id, SyncTask::Upload(upload_data)); - UploadedTimeline::FailedAndRescheduled(anyhow::anyhow!( - "Errors appeared during layer uploads: {:?}", - errors - )) - } -} + let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))?; -enum UploadError { - MissingLocalFile(PathBuf, anyhow::Error), - Other(anyhow::Error), -} - -#[cfg(test)] -mod tests { - use std::{ - collections::{BTreeSet, HashSet}, - num::NonZeroUsize, - }; - - use remote_storage::{LocalFs, RemoteStorage}; - use tempfile::tempdir; - use utils::lsn::Lsn; - - use crate::{ - storage_sync::{ - index::RelativePath, - test_utils::{create_local_timeline, dummy_metadata}, - }, - tenant::harness::{TenantHarness, TIMELINE_ID}, - }; - - use super::{upload_index_part, *}; - - #[tokio::test] - async fn regular_layer_upload() -> anyhow::Result<()> { - let harness = TenantHarness::create("regular_layer_upload")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - - let layer_files = ["a", "b"]; - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_path_buf(), - harness.conf.workdir.clone(), - )?); - let local_storage = storage.as_local().unwrap(); - let current_retries = 3; - let metadata = dummy_metadata(Lsn(0x30)); - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - let mut timeline_upload = - create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?; - timeline_upload.metadata = None; - - assert!( - local_storage.list().await?.is_empty(), - "Storage should be empty before any uploads are made" - ); - - let upload_result = upload_timeline_layers( - &storage, - &sync_queue, - None, - sync_id, - SyncData::new(current_retries, timeline_upload.clone()), - ) - .await; - - let upload_data = match upload_result { - UploadedTimeline::Successful(upload_data) => upload_data, - wrong_result => { - panic!("Expected a successful upload for timeline, but got: {wrong_result:?}") - } - }; - - assert_eq!( - current_retries, upload_data.retries, - "On successful upload, retries are not expected to change" - ); - let upload = &upload_data.data; - assert!( - upload.layers_to_upload.is_empty(), - "Successful upload should have no layers left to upload" - ); - assert_eq!( - upload - .uploaded_layers - .keys() - .cloned() - .collect::>(), - layer_files - .iter() - .map(|layer_file| local_timeline_path.join(layer_file)) - .collect(), - "Successful upload should have all layers uploaded" - ); - assert_eq!( - upload.metadata, None, - "Successful upload without metadata should not have it returned either" - ); - - let storage_files = local_storage.list().await?; - assert_eq!( - storage_files.len(), - layer_files.len(), - "All layers should be uploaded" - ); - assert_eq!( - storage_files - .into_iter() - .map(|storage_path| local_storage.local_path(&storage_path)) - .collect::>>()?, - layer_files - .into_iter() - .map(|file| local_timeline_path.join(file)) - .collect(), - "Uploaded files should match with the local ones" - ); - - Ok(()) - } - - // Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario. - #[tokio::test] - async fn layer_upload_after_local_fs_update() -> anyhow::Result<()> { - let harness = TenantHarness::create("layer_upload_after_local_fs_update")?; - let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap()); - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - - let layer_files = ["a1", "b1"]; - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_owned(), - harness.conf.workdir.clone(), - )?); - let local_storage = storage.as_local().unwrap(); - let current_retries = 5; - let metadata = dummy_metadata(Lsn(0x40)); - - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - let layers_to_upload = { - let mut layers = layer_files.to_vec(); - layers.push("layer_to_remove"); - layers - }; - let timeline_upload = - create_local_timeline(&harness, TIMELINE_ID, &layers_to_upload, metadata.clone()) - .await?; - assert!( - local_storage.list().await?.is_empty(), - "Storage should be empty before any uploads are made" - ); - - fs::remove_file(local_timeline_path.join("layer_to_remove")).await?; - - let upload_result = upload_timeline_layers( - &storage, - &sync_queue, - None, - sync_id, - SyncData::new(current_retries, timeline_upload.clone()), - ) - .await; - - let upload_data = match upload_result { - UploadedTimeline::Successful(upload_data) => upload_data, - wrong_result => panic!( - "Expected a successful after local fs upload for timeline, but got: {wrong_result:?}" - ), - }; - - assert_eq!( - current_retries, upload_data.retries, - "On successful upload, retries are not expected to change" - ); - let upload = &upload_data.data; - assert!( - upload.layers_to_upload.is_empty(), - "Successful upload should have no layers left to upload, even those that were removed from the local fs" - ); - assert_eq!( - upload - .uploaded_layers - .keys() - .cloned() - .collect::>(), - layer_files - .iter() - .map(|layer_file| local_timeline_path.join(layer_file)) - .collect(), - "Successful upload should have all layers uploaded" - ); - assert_eq!( - upload.metadata, - Some(metadata), - "Successful upload should not change its metadata" - ); - - let storage_files = local_storage.list().await?; - assert_eq!( - storage_files.len(), - layer_files.len(), - "All layers should be uploaded" - ); - assert_eq!( - storage_files - .into_iter() - .map(|storage_path| local_storage.local_path(&storage_path)) - .collect::>>()?, - layer_files - .into_iter() - .map(|file| local_timeline_path.join(file)) - .collect(), - "Uploaded files should match with the local ones" - ); - - Ok(()) - } - - #[tokio::test] - async fn test_upload_index_part() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_upload_index_part")?; - let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID); - - let storage = GenericRemoteStorage::new(LocalFs::new( - tempdir()?.path().to_owned(), - harness.conf.workdir.clone(), - )?); - let local_storage = storage.as_local().unwrap(); - let metadata = dummy_metadata(Lsn(0x40)); - let local_timeline_path = harness.timeline_path(&TIMELINE_ID); - - let index_part = IndexPart::new( - HashSet::from([ - RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?, - RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?, - ]), - HashSet::from([RelativePath::new( - &local_timeline_path, - local_timeline_path.join("three"), - )?]), - metadata.disk_consistent_lsn(), - metadata.to_bytes()?, - ); - - assert!( - local_storage.list().await?.is_empty(), - "Storage should be empty before any uploads are made" - ); - upload_index_part(harness.conf, &storage, sync_id, index_part.clone()).await?; - - let storage_files = local_storage.list().await?; - assert_eq!( - storage_files.len(), - 1, - "Should have only the index part file uploaded" - ); - - let index_part_path = storage_files.first().unwrap(); - assert_eq!( - index_part_path.object_name(), - Some(IndexPart::FILE_NAME), - "Remote index part should have the correct name" - ); - let remote_index_part: IndexPart = serde_json::from_slice( - &fs::read(local_storage.resolve_in_storage(index_part_path)?).await?, - )?; - assert_eq!( - index_part, remote_index_part, - "Remote index part should match the local one" - ); + storage + .upload(Box::new(source_file), fs_size, &storage_path, None) + .await + .with_context(|| { + format!( + "Failed to upload a layer from local path '{}'", + source_path.display() + ) + })?; Ok(()) } + .await } diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index dad6e0039d..86d1266f09 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -197,8 +197,8 @@ pub enum TaskKind { // Task that flushes frozen in-memory layers to disk LayerFlushTask, - // Task that manages the remote upload queue - StorageSync, + // Task that uploads a file to remote storage + RemoteUploadTask, // task that handles the initial downloading of all tenants InitialLoad, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 11845ebb48..5647a0c345 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -15,6 +15,7 @@ use anyhow::{bail, Context}; use bytes::Bytes; use futures::Stream; use pageserver_api::models::TimelineState; +use remote_storage::GenericRemoteStorage; use tokio::sync::watch; use tokio_util::io::StreamReader; use tokio_util::io::SyncIoBridge; @@ -25,6 +26,7 @@ use std::cmp::min; use std::collections::hash_map::Entry; use std::collections::BTreeSet; use std::collections::HashMap; +use std::ffi::OsStr; use std::fs; use std::fs::File; use std::fs::OpenOptions; @@ -43,13 +45,20 @@ use std::time::{Duration, Instant}; use self::metadata::TimelineMetadata; use crate::config::PageServerConf; +use crate::config::TIMELINE_UNINIT_MARK_SUFFIX; use crate::import_datadir; use crate::metrics::{remove_tenant_metrics, STORAGE_TIME}; use crate::repository::GcResult; -use crate::storage_sync::index::RemoteIndex; +use crate::storage_sync::create_remote_timeline_client; +use crate::storage_sync::index::IndexPart; +use crate::storage_sync::list_remote_timelines; +use crate::storage_sync::RemoteTimelineClient; use crate::task_mgr; +use crate::task_mgr::TaskKind; +use crate::tenant::metadata::load_metadata; use crate::tenant_config::TenantConfOpt; use crate::virtual_file::VirtualFile; +use crate::walredo::PostgresRedoManager; use crate::walredo::WalRedoManager; use crate::{CheckpointConfig, TEMP_FILE_SUFFIX}; pub use pageserver_api::models::TenantState; @@ -122,11 +131,7 @@ pub struct Tenant { walredo_mgr: Arc, // provides access to timeline data sitting in the remote storage - // supposed to be used for retrieval of remote consistent lsn in walreceiver - remote_index: RemoteIndex, - - /// Makes every timeline to backup their files to remote storage. - upload_layers: bool, + remote_storage: Option, /// Cached logical sizes updated updated on each [`Tenant::gather_size_inputs`]. cached_logical_sizes: tokio::sync::Mutex>, @@ -353,6 +358,468 @@ impl Drop for TimelineUninitMark { /// A repository corresponds to one .neon directory. One repository holds multiple /// timelines, forked off from the same initial call to 'initdb'. impl Tenant { + /// Yet another helper for timeline initialization. + /// Contains common part for `load_remote_timeline` and `load_remote_timeline` + async fn setup_timeline( + &self, + timeline_id: TimelineId, + remote_client: Option, + index_part: Option<&IndexPart>, + metadata: TimelineMetadata, + ancestor: Option>, + first_save: bool, + ) -> anyhow::Result<()> { + let tenant_id = self.tenant_id; + + let timeline = { + // avoiding holding it across awaits + let mut timelines_accessor = self.timelines.lock().unwrap(); + if timelines_accessor.contains_key(&timeline_id) { + anyhow::bail!( + "Timeline {tenant_id}/{timeline_id} already exists in the tenant map" + ); + } + + let dummy_timeline = self.create_timeline_data( + timeline_id, + metadata.clone(), + ancestor.clone(), + remote_client, + )?; + + let timeline = UninitializedTimeline { + owning_tenant: self, + timeline_id, + raw_timeline: Some((Arc::new(dummy_timeline), TimelineUninitMark::dummy())), + }; + + // FIXME reminder, it will start walreceiver which may be undesirable if + // initialization of other timelines fail, may be fine if we go to broken state + match timeline.initialize_with_lock(&mut timelines_accessor, true) { + Ok(initialized_timeline) => { + timelines_accessor.insert(timeline_id, initialized_timeline.clone()); + Ok(initialized_timeline) + } + Err(e) => { + error!("Failed to initialize timeline {tenant_id}/{timeline_id}: {e:?}"); + // FIXME using None is a hack, it wont hurt, just ugly. + // Ideally initialize_with_lock error should return timeline in the error + // Or return ownership of itself completely so somethin like into_broken + // can be called directly on Uninitielized timeline + // also leades to redundant .clone + let broken_timeline = self + .create_timeline_data(timeline_id, metadata.clone(), ancestor, None) + .with_context(|| { + format!( + "Failed to crate broken timeline data for {tenant_id}/{timeline_id}" + ) + })?; + broken_timeline.set_state(TimelineState::Broken); + timelines_accessor.insert(timeline_id, Arc::new(broken_timeline)); + Err(e) + } + } + }?; + + if self.remote_storage.is_some() { + // Reconcile local state with remote storage, downloading anything that's + // missing locally, and scheduling uploads for anything that's missing + // in remote storage. + timeline + .reconcile_with_remote(metadata, index_part, first_save) + .await? + } + + Ok(()) + } + + /// + /// Attach a tenant that's available in cloud storage. + /// + /// This returns quickly, after just creating the in-memory object + /// Tenant struct and launching a background task to download + /// the remote index files. On return, the tenant is most likely still in + /// Attaching state, and it will become Active once the background task + /// finishes. You can use wait_until_active() to wait for the task to + /// complete. + /// + pub fn spawn_attach( + conf: &'static PageServerConf, + tenant_id: TenantId, + remote_storage: &GenericRemoteStorage, + ) -> anyhow::Result> { + // FIXME: where to get tenant config when attaching? This will just fill in + // the defaults + let tenant_conf = Self::load_tenant_config(conf, tenant_id)?; + + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let tenant = Arc::new(Tenant::new( + TenantState::Attaching, + conf, + tenant_conf, + wal_redo_manager, + tenant_id, + Some(remote_storage.clone()), + )); + + // Do all the hard work in the background + let tenant_clone = Arc::clone(&tenant); + + task_mgr::spawn( + &tokio::runtime::Handle::current(), + TaskKind::Attach, + Some(tenant_id), + None, + "attach tenant", + false, + async move { + match tenant_clone.attach().await { + Ok(_) => {} + Err(e) => { + tenant_clone.set_state(TenantState::Broken); + error!("error attaching tenant: {:?}", e); + } + } + Ok(()) + }, + ); + Ok(tenant) + } + + /// + /// Background task that downloads all data for a tenant and brings it to Active state. + /// + #[instrument(skip(self), fields(tenant_id=%self.tenant_id))] + async fn attach(self: &Arc) -> anyhow::Result<()> { + // Get list of remote timelines + // download index files for every tenant timeline + info!("listing remote timelines"); + + let remote_storage = self + .remote_storage + .as_ref() + .ok_or_else(|| anyhow::anyhow!("cannot attach without remote storage"))?; + + let remote_timelines = + list_remote_timelines(remote_storage, self.conf, self.tenant_id).await?; + + info!("found {} timelines", remote_timelines.len()); + + let mut timeline_ancestors: HashMap = HashMap::new(); + let mut index_parts: HashMap = HashMap::new(); + for (timeline_id, index_part) in remote_timelines.iter() { + let remote_metadata = index_part.parse_metadata().with_context(|| { + format!( + "Failed to parse metadata file from remote storage for tenant {}", + self.tenant_id + ) + })?; + timeline_ancestors.insert(*timeline_id, remote_metadata); + index_parts.insert(*timeline_id, index_part); + } + + let sorted_timelines = tree_sort_timelines(timeline_ancestors)?; + + // For every timeline, download the metadata file, scan the local directory, + // and build a layer map that contains an entry for each remote and local + // layer file. + for (timeline_id, _metadata) in sorted_timelines { + // TODO again handle early failure + self.load_remote_timeline( + timeline_id, + index_parts[&timeline_id], + remote_storage.clone(), + ) + .instrument(info_span!("load_remote_timeline", timeline = %timeline_id)) + .await? + } + + // FIXME: Check if the state has changed to Stopping while we were downloading stuff + // We're ready for business. + // Change to active state under the hood spawns background loops + // The loops will shut themselves down when they notice that the tenant is inactive. + self.set_state(TenantState::Active { + background_jobs_running: true, + }); + + info!("Done"); + + Ok(()) + } + + async fn load_remote_timeline( + &self, + timeline_id: TimelineId, + index_part: &IndexPart, + remote_storage: GenericRemoteStorage, + ) -> anyhow::Result<()> { + info!("downloading index file for timeline {}", timeline_id); + tokio::fs::create_dir_all(self.conf.timeline_path(&timeline_id, &self.tenant_id)) + .await + .context("Failed to create new timeline directory")?; + + let remote_metadata = index_part.parse_metadata().with_context(|| { + format!( + "Failed to parse metadata file from remote storage for tenant {}", + self.tenant_id + ) + })?; + + let remote_client = + create_remote_timeline_client(remote_storage, self.conf, self.tenant_id, timeline_id)?; + + let ancestor = if let Some(ancestor_id) = remote_metadata.ancestor_timeline() { + let timelines = self.timelines.lock().unwrap(); + Some(Arc::clone(timelines.get(&ancestor_id).ok_or_else( + || { + anyhow::anyhow!( + "cannot find ancestor timeline {ancestor_id} for timeline {timeline_id}" + ) + }, + )?)) + } else { + None + }; + + self.setup_timeline( + timeline_id, + Some(remote_client), + Some(index_part), + remote_metadata, + ancestor, + true, + ) + .await + } + + /// + /// Load a tenant that's available on local disk + /// + /// This is used at pageserver startup, to rebuild the in-memory + /// structures from on-disk state. This is similar to attaching a tenant, + /// but the index files already exist on local disk, as well as some layer + /// files. + /// + /// If the loading fails for some reason, the Tenant will go into Broken + /// state. + /// + #[instrument(skip(conf, remote_storage), fields(tenant_id=%tenant_id))] + pub fn spawn_load( + conf: &'static PageServerConf, + tenant_id: TenantId, + remote_storage: Option, + ) -> anyhow::Result> { + // FIXME: also go into Broken state if this fails + let tenant_conf = Self::load_tenant_config(conf, tenant_id)?; + + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let tenant = Tenant::new( + TenantState::Loading, + conf, + tenant_conf, + wal_redo_manager, + tenant_id, + remote_storage, + ); + let tenant = Arc::new(tenant); + + // Do all the hard work in a background task + let tenant_clone = Arc::clone(&tenant); + + let _ = task_mgr::spawn( + &tokio::runtime::Handle::current(), + TaskKind::InitialLoad, + Some(tenant_id), + None, + "initial tenant load", + false, + async move { + match tenant_clone.load().await { + Ok(()) => {} + Err(err) => { + tenant_clone.set_state(TenantState::Broken); + error!("could not load tenant {tenant_id}: {err:?}"); + } + } + info!("initial load for tenant {tenant_id} finished!"); + Ok(()) + }, + ); + + info!("spawned load into background"); + + Ok(tenant) + } + + /// + /// Background task to load in-memory data structures for this tenant, from + /// files on disk. Used at pageserver startup. + /// + #[instrument(skip(self), fields(tenant_id=%self.tenant_id))] + async fn load(self: &Arc) -> anyhow::Result<()> { + info!("loading tenant task"); + + // TODO split this into two functions, scan and actual load + + // Load in-memory state to reflect the local files on disk + // + // Scan the directory, peek into the metadata file of each timeline, and + // collect a list of timelines and their ancestors. + let mut timelines_to_load: HashMap = HashMap::new(); + let timelines_dir = self.conf.timelines_path(&self.tenant_id); + for entry in std::fs::read_dir(&timelines_dir).with_context(|| { + format!( + "Failed to list timelines directory for tenant {}", + self.tenant_id + ) + })? { + let entry = entry?; + let timeline_dir = entry.path(); + + if crate::is_temporary(&timeline_dir) { + info!( + "Found temporary timeline directory, removing: {}", + timeline_dir.display() + ); + if let Err(e) = std::fs::remove_dir_all(&timeline_dir) { + error!( + "Failed to remove temporary directory '{}': {:?}", + timeline_dir.display(), + e + ); + } + } else if is_uninit_mark(&timeline_dir) { + let timeline_uninit_mark_file = &timeline_dir; + info!( + "Found an uninit mark file {}, removing the timeline and its uninit mark", + timeline_uninit_mark_file.display() + ); + let timeline_id = timeline_uninit_mark_file + .file_stem() + .and_then(OsStr::to_str) + .unwrap_or_default() + .parse::() + .with_context(|| { + format!( + "Could not parse timeline id out of the timeline uninit mark name {}", + timeline_uninit_mark_file.display() + ) + })?; + let timeline_dir = self.conf.timeline_path(&timeline_id, &self.tenant_id); + if let Err(e) = + remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file) + { + error!("Failed to clean up uninit marked timeline: {e:?}"); + } + } else { + let timeline_id = timeline_dir + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .parse::() + .with_context(|| { + format!( + "Could not parse timeline id out of the timeline dir name {}", + timeline_dir.display() + ) + })?; + let timeline_uninit_mark_file = self + .conf + .timeline_uninit_mark_file_path(self.tenant_id, timeline_id); + if timeline_uninit_mark_file.exists() { + info!( + "Found an uninit mark file for timeline {}/{}, removing the timeline and its uninit mark", + self.tenant_id, timeline_id + ); + if let Err(e) = + remove_timeline_and_uninit_mark(&timeline_dir, &timeline_uninit_mark_file) + { + error!("Failed to clean up uninit marked timeline: {e:?}"); + } + continue; + } + + // TODO There was also remove if dir is empty logic + + let file_name = entry.file_name(); + if let Ok(timeline_id) = + file_name.to_str().unwrap_or_default().parse::() + { + let metadata = load_metadata(self.conf, timeline_id, self.tenant_id) + .context("failed to load metadata")?; + timelines_to_load.insert(timeline_id, metadata); + } else { + // A file or directory that doesn't look like a timeline ID + warn!( + "unexpected file or directory in timelines directory: {}", + file_name.to_string_lossy() + ); + } + } + } + + // Sort the array of timeline IDs into tree-order, so that parent comes before + // all its children. + let sorted_timelines = tree_sort_timelines(timelines_to_load)?; + // FIXME if we init some of the timelines and one of them is broken we + // potentially will end up with partially launched wal receivers. + // 1. Either move it out of initialize_with_lock + // 2. Or pass it as a flag + // FIXME original collect_timeline_files contained one more check: + // 1. "Timeline has no ancestor and no layer files" + + for (timeline_id, metadata) in sorted_timelines { + // FIXME should we fail load of whole tenant if one timeline failed? + // consider branch hierarchy. Maybe set one to broken and others to Paused or something + self.load_local_timeline(timeline_id, metadata).await?; + } + + // We're ready for business. + // Change to active state under the hood spawns background loops + // The loops will shut themselves down when they notice that the tenant is inactive. + self.set_state(TenantState::Active { + background_jobs_running: true, + }); + + info!("Done"); + + Ok(()) + } + + /// Subroutine of `load_tenant`, to load an individual timeline + /// + /// NB: The parent is assumed to be already loaded! + #[instrument(skip(self, metadata), fields(tenant_id=%self.tenant_id, timeline_id=%timeline_id))] + async fn load_local_timeline( + &self, + timeline_id: TimelineId, + metadata: TimelineMetadata, + ) -> anyhow::Result<()> { + let ancestor = if let Some(ancestor_timeline_id) = metadata.ancestor_timeline() { + let ancestor_timeline = self.get_timeline(ancestor_timeline_id, false) + .with_context(|| anyhow::anyhow!("cannot find ancestor timeline {ancestor_timeline_id} for timeline {timeline_id}"))?; + Some(ancestor_timeline) + } else { + None + }; + + let remote_client = self + .remote_storage + .as_ref() + .map(|remote_storage| { + create_remote_timeline_client( + remote_storage.clone(), + self.conf, + self.tenant_id, + timeline_id, + ) + }) + .transpose()?; + + self.setup_timeline(timeline_id, remote_client, None, metadata, ancestor, false) + .await + } + pub fn tenant_id(&self) -> TenantId { self.tenant_id } @@ -437,7 +904,7 @@ impl Tenant { /// a new unique ID is generated. pub async fn create_timeline( &self, - new_timeline_id: Option, + new_timeline_id: TimelineId, ancestor_timeline_id: Option, mut ancestor_start_lsn: Option, pg_version: u32, @@ -447,8 +914,6 @@ impl Tenant { "Cannot create timelines on inactive tenant" ); - let new_timeline_id = new_timeline_id.unwrap_or_else(TimelineId::generate); - if self.get_timeline(new_timeline_id, false).is_ok() { debug!("timeline {new_timeline_id} already exists"); return Ok(None); @@ -624,11 +1089,6 @@ impl Tenant { Ok(()) } - /// Allows to retrieve remote timeline index from the tenant. Used in walreceiver to grab remote consistent lsn. - pub fn get_remote_index(&self) -> &RemoteIndex { - &self.remote_index - } - pub fn current_state(&self) -> TenantState { *self.state.borrow() } @@ -688,6 +1148,8 @@ impl Tenant { timeline.set_state(TimelineState::Suspended); } } + TenantState::Loading => { /* Should we do something here? */ } + TenantState::Attaching => { /* Should we do something here? */ } } } } @@ -822,6 +1284,7 @@ impl Tenant { new_timeline_id: TimelineId, new_metadata: TimelineMetadata, ancestor: Option>, + remote_client: Option, ) -> anyhow::Result { if let Some(ancestor_timeline_id) = new_metadata.ancestor_timeline() { anyhow::ensure!( @@ -839,20 +1302,20 @@ impl Tenant { new_timeline_id, self.tenant_id, Arc::clone(&self.walredo_mgr), - self.upload_layers, + remote_client, pg_version, )) } pub(super) fn new( + state: TenantState, conf: &'static PageServerConf, tenant_conf: TenantConfOpt, walredo_mgr: Arc, tenant_id: TenantId, - remote_index: RemoteIndex, - upload_layers: bool, + remote_storage: Option, ) -> Tenant { - let (state, _) = watch::channel(TenantState::Paused); + let (state, _) = watch::channel(state); Tenant { tenant_id, conf, @@ -860,8 +1323,7 @@ impl Tenant { timelines: Mutex::new(HashMap::new()), gc_cs: Mutex::new(()), walredo_mgr, - remote_index, - upload_layers, + remote_storage, state, cached_logical_sizes: tokio::sync::Mutex::new(HashMap::new()), } @@ -1354,11 +1816,25 @@ impl Tenant { ) -> anyhow::Result { let tenant_id = self.tenant_id; + let remote_client = if let Some(remote_storage) = self.remote_storage.as_ref() { + let remote_client = create_remote_timeline_client( + remote_storage.clone(), + self.conf, + tenant_id, + new_timeline_id, + )?; + remote_client.init_upload_queue_empty(&new_metadata)?; + Some(remote_client) + } else { + None + }; + match self.create_timeline_files( &uninit_mark.timeline_path, new_timeline_id, new_metadata, ancestor, + remote_client, ) { Ok(new_timeline) => { if init_layers { @@ -1388,9 +1864,15 @@ impl Tenant { new_timeline_id: TimelineId, new_metadata: TimelineMetadata, ancestor: Option>, + remote_client: Option, ) -> anyhow::Result { let timeline_data = self - .create_timeline_data(new_timeline_id, new_metadata.clone(), ancestor) + .create_timeline_data( + new_timeline_id, + new_metadata.clone(), + ancestor, + remote_client, + ) .context("Failed to create timeline data structure")?; crashsafe::create_dir_all(timeline_path).context("Failed to create timeline directory")?; @@ -1450,68 +1932,6 @@ impl Tenant { Ok(uninit_mark) } - pub(super) fn init_attach_timelines( - &self, - timelines: HashMap, - ) -> anyhow::Result<()> { - let sorted_timelines = if timelines.len() == 1 { - timelines.into_iter().collect() - } else if !timelines.is_empty() { - tree_sort_timelines(timelines)? - } else { - warn!("No timelines to attach received"); - return Ok(()); - }; - - let tenant_id = self.tenant_id; - let mut timelines_accessor = self.timelines.lock().unwrap(); - for (timeline_id, metadata) in sorted_timelines { - info!( - "Attaching timeline {}/{} pg_version {}", - tenant_id, - timeline_id, - metadata.pg_version() - ); - - if timelines_accessor.contains_key(&timeline_id) { - warn!("Timeline {tenant_id}/{timeline_id} already exists in the tenant map, skipping its initialization"); - continue; - } - - let ancestor = metadata - .ancestor_timeline() - .and_then(|ancestor_timeline_id| timelines_accessor.get(&ancestor_timeline_id)) - .cloned(); - let dummy_timeline = self - .create_timeline_data(timeline_id, metadata.clone(), ancestor.clone()) - .with_context(|| { - format!("Failed to crate dummy timeline data for {tenant_id}/{timeline_id}") - })?; - let timeline = UninitializedTimeline { - owning_tenant: self, - timeline_id, - raw_timeline: Some((Arc::new(dummy_timeline), TimelineUninitMark::dummy())), - }; - match timeline.initialize_with_lock(&mut timelines_accessor, true) { - Ok(initialized_timeline) => { - timelines_accessor.insert(timeline_id, initialized_timeline); - } - Err(e) => { - error!("Failed to initialize timeline {tenant_id}/{timeline_id}: {e:?}"); - let broken_timeline = self - .create_timeline_data(timeline_id, metadata, ancestor) - .with_context(|| { - format!("Failed to crate broken timeline data for {tenant_id}/{timeline_id}") - })?; - broken_timeline.set_state(TimelineState::Broken); - timelines_accessor.insert(timeline_id, Arc::new(broken_timeline)); - } - } - } - - Ok(()) - } - /// Gathers inputs from all of the timelines to produce a sizing model input. /// /// Future is cancellation safe. Only one calculation can be running at once per tenant. @@ -1532,6 +1952,42 @@ impl Tenant { } } +fn is_uninit_mark(path: &Path) -> bool { + match path.file_name() { + Some(name) => name + .to_string_lossy() + .ends_with(TIMELINE_UNINIT_MARK_SUFFIX), + None => false, + } +} + +fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> { + fs::remove_dir_all(&timeline_dir) + .or_else(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + // we can leave the uninit mark without a timeline dir, + // just remove the mark then + Ok(()) + } else { + Err(e) + } + }) + .with_context(|| { + format!( + "Failed to remove unit marked timeline directory {}", + timeline_dir.display() + ) + })?; + fs::remove_file(&uninit_mark).with_context(|| { + format!( + "Failed to remove timeline uninit mark file {}", + uninit_mark.display() + ) + })?; + + Ok(()) +} + /// Create the cluster temporarily in 'initdbpath' directory inside the repository /// to get bootstrap data for timeline initialization. fn run_initdb( @@ -1627,7 +2083,6 @@ pub mod harness { use std::{fs, path::PathBuf}; use utils::lsn::Lsn; - use crate::storage_sync::index::RemoteIndex; use crate::{ config::PageServerConf, repository::Key, @@ -1726,21 +2181,23 @@ pub mod harness { }) } - pub fn load(&self) -> Tenant { - self.try_load().expect("failed to load test tenant") + pub async fn load(&self) -> Arc { + self.try_load().await.expect("failed to load test tenant") } - pub fn try_load(&self) -> anyhow::Result { + pub async fn try_load(&self) -> anyhow::Result> { let walredo_mgr = Arc::new(TestRedoManager); - let tenant = Tenant::new( + let tenant = Arc::new(Tenant::new( + TenantState::Active { + background_jobs_running: false, + }, self.conf, TenantConfOpt::from(self.tenant_conf), walredo_mgr, self.tenant_id, - RemoteIndex::default(), - false, - ); + None, + )); // populate tenant with locally available timelines let mut timelines_to_load = HashMap::new(); for timeline_dir_entry in fs::read_dir(self.conf.timelines_path(&self.tenant_id)) @@ -1757,10 +2214,8 @@ pub mod harness { let timeline_metadata = load_metadata(self.conf, timeline_id, self.tenant_id)?; timelines_to_load.insert(timeline_id, timeline_metadata); } - tenant.init_attach_timelines(timelines_to_load)?; - tenant.set_state(TenantState::Active { - background_jobs_running: false, - }); + // FIXME starts background jobs + tenant.load().await?; Ok(tenant) } @@ -1770,26 +2225,6 @@ pub mod harness { } } - fn load_metadata( - conf: &'static PageServerConf, - timeline_id: TimelineId, - tenant_id: TenantId, - ) -> anyhow::Result { - let metadata_path = conf.metadata_path(timeline_id, tenant_id); - let metadata_bytes = std::fs::read(&metadata_path).with_context(|| { - format!( - "Failed to read metadata bytes from path {}", - metadata_path.display() - ) - })?; - TimelineMetadata::from_bytes(&metadata_bytes).with_context(|| { - format!( - "Failed to parse metadata bytes from path {}", - metadata_path.display() - ) - }) - } - // Mock WAL redo manager that doesn't do much pub struct TestRedoManager; @@ -1836,9 +2271,9 @@ mod tests { static TEST_KEY: Lazy = Lazy::new(|| Key::from_slice(&hex!("112222222233333333444444445500000001"))); - #[test] - fn test_basic() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_basic")?.load(); + #[tokio::test] + async fn test_basic() -> anyhow::Result<()> { + let tenant = TenantHarness::create("test_basic")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -1860,9 +2295,11 @@ mod tests { Ok(()) } - #[test] - fn no_duplicate_timelines() -> anyhow::Result<()> { - let tenant = TenantHarness::create("no_duplicate_timelines")?.load(); + #[tokio::test] + async fn no_duplicate_timelines() -> anyhow::Result<()> { + let tenant = TenantHarness::create("no_duplicate_timelines")? + .load() + .await; let _ = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -1891,9 +2328,9 @@ mod tests { /// /// Test branch creation /// - #[test] - fn test_branch() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_branch")?.load(); + #[tokio::test] + async fn test_branch() -> anyhow::Result<()> { + let tenant = TenantHarness::create("test_branch")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -1990,7 +2427,8 @@ mod tests { async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { let tenant = TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? - .load(); + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2020,10 +2458,11 @@ mod tests { Ok(()) } - #[test] - fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { - let tenant = - TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")?.load(); + #[tokio::test] + async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { + let tenant = TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? + .load() + .await; tenant .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION)? @@ -2069,8 +2508,9 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { - let tenant = - TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")?.load(); + let tenant = TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2090,8 +2530,9 @@ mod tests { } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let tenant = - TenantHarness::create("test_parent_keeps_data_forever_after_branching")?.load(); + let tenant = TenantHarness::create("test_parent_keeps_data_forever_after_branching")? + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2123,7 +2564,7 @@ mod tests { const TEST_NAME: &str = "timeline_load"; let harness = TenantHarness::create(TEST_NAME)?; { - let tenant = harness.load(); + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION)? .initialize()?; @@ -2131,7 +2572,7 @@ mod tests { tline.checkpoint(CheckpointConfig::Forced).await?; } - let tenant = harness.load(); + let tenant = harness.load().await; tenant .get_timeline(TIMELINE_ID, true) .expect("cannot load timeline"); @@ -2145,7 +2586,7 @@ mod tests { let harness = TenantHarness::create(TEST_NAME)?; // create two timelines { - let tenant = harness.load(); + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2164,7 +2605,7 @@ mod tests { } // check that both of them are initially unloaded - let tenant = harness.load(); + let tenant = harness.load().await; // check that both, child and ancestor are loaded let _child_tline = tenant @@ -2178,11 +2619,11 @@ mod tests { Ok(()) } - #[test] - fn corrupt_metadata() -> anyhow::Result<()> { + #[tokio::test] + async fn corrupt_metadata() -> anyhow::Result<()> { const TEST_NAME: &str = "corrupt_metadata"; let harness = TenantHarness::create(TEST_NAME)?; - let tenant = harness.load(); + let tenant = harness.load().await; tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? @@ -2198,7 +2639,7 @@ mod tests { metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness.try_load().err().expect("should fail"); + let err = harness.try_load().await.err().expect("should fail"); assert!(err .to_string() .starts_with("Failed to parse metadata bytes from path")); @@ -2222,7 +2663,7 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_images")?.load(); + let tenant = TenantHarness::create("test_images")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2274,7 +2715,7 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_bulk_insert")?.load(); + let tenant = TenantHarness::create("test_bulk_insert")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2316,7 +2757,7 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_random_updates")?.load(); + let tenant = TenantHarness::create("test_random_updates")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2388,7 +2829,9 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_branches")?.load(); + let tenant = TenantHarness::create("test_traverse_branches")? + .load() + .await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2467,9 +2910,11 @@ mod tests { Ok(()) } - #[test] - fn test_traverse_ancestors() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_ancestors")?.load(); + #[tokio::test] + async fn test_traverse_ancestors() -> anyhow::Result<()> { + let tenant = TenantHarness::create("test_traverse_ancestors")? + .load() + .await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 3fb9ccd936..6f22fbd5d2 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -264,6 +264,26 @@ pub fn save_metadata( Ok(()) } +pub fn load_metadata( + conf: &'static PageServerConf, + timeline_id: TimelineId, + tenant_id: TenantId, +) -> anyhow::Result { + let metadata_path = conf.metadata_path(timeline_id, tenant_id); + let metadata_bytes = std::fs::read(&metadata_path).with_context(|| { + format!( + "Failed to read metadata bytes from path {}", + metadata_path.display() + ) + })?; + TimelineMetadata::from_bytes(&metadata_bytes).with_context(|| { + format!( + "Failed to parse metadata bytes from path {}", + metadata_path.display() + ) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 0b2f7876db..0a532fcea7 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -6,6 +6,7 @@ use fail::fail_point; use itertools::Itertools; use once_cell::sync::OnceCell; use pageserver_api::models::TimelineState; +use remote_storage::DownloadError; use tokio::sync::watch; use tokio::task::spawn_blocking; use tracing::*; @@ -14,11 +15,13 @@ use std::cmp::{max, min, Ordering}; use std::collections::{HashMap, HashSet}; use std::fs; use std::ops::{Deref, Range}; -use std::path::PathBuf; -use std::sync::atomic::{self, AtomicBool, AtomicI64, Ordering as AtomicOrdering}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use std::time::{Duration, Instant, SystemTime}; +use crate::storage_sync::index::{IndexPart, RelativePath}; +use crate::storage_sync::RemoteTimelineClient; use crate::tenant::{ delta_layer::{DeltaLayer, DeltaLayerWriter}, ephemeral_file::is_ephemeral_file, @@ -56,10 +59,7 @@ use crate::walreceiver::{is_etcd_client_initialized, spawn_connection_manager_ta use crate::walredo::WalRedoManager; use crate::CheckpointConfig; use crate::ZERO_PAGE; -use crate::{ - page_cache, - storage_sync::{self, index::LayerFileMetadata}, -}; +use crate::{page_cache, storage_sync::index::LayerFileMetadata}; pub struct Timeline { conf: &'static PageServerConf, @@ -79,6 +79,13 @@ pub struct Timeline { // WAL redo manager walredo_mgr: Arc, + /// Remote storage client. + /// + /// If Some, use it to upload all newly created layers to the remote storage, + /// and keep remote metadata file in sync. In the future, also use it to download + /// layer files on-demand. + pub remote_client: Option>, + // What page versions do we hold in the repository? If we get a // request > last_record_lsn, we need to wait until we receive all // the WAL up to the request. The SeqWait provides functions for @@ -111,9 +118,6 @@ pub struct Timeline { // Metrics metrics: TimelineMetrics, - /// If `true`, will backup its files that appear after each checkpointing to the remote storage. - upload_layers: AtomicBool, - /// Ensures layers aren't frozen by checkpointer between /// [`Timeline::get_layer_for_write`] and layer reads. /// Locked automatically by [`TimelineWriter`] and checkpointer. @@ -400,6 +404,14 @@ impl Timeline { self.disk_consistent_lsn.load() } + pub fn get_remote_consistent_lsn(&self) -> Option { + if let Some(remote_client) = &self.remote_client { + remote_client.last_uploaded_consistent_lsn() + } else { + None + } + } + /// Get the physical size of the timeline at the latest LSN pub fn get_physical_size(&self) -> u64 { self.metrics.current_physical_size_gauge.get() @@ -546,15 +558,10 @@ impl Timeline { // 2. Create new image layers for partitions that have been modified // "enough". let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?; - if !layer_paths_to_upload.is_empty() - && self.upload_layers.load(atomic::Ordering::Relaxed) - { - storage_sync::schedule_layer_upload( - self.tenant_id, - self.timeline_id, - layer_paths_to_upload, - None, - ); + if let Some(remote_client) = &self.remote_client { + for (path, layer_metadata) in layer_paths_to_upload { + remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; + } } // 3. Compact @@ -719,7 +726,7 @@ impl Timeline { timeline_id: TimelineId, tenant_id: TenantId, walredo_mgr: Arc, - upload_layers: bool, + remote_client: Option, pg_version: u32, ) -> Self { let disk_consistent_lsn = metadata.disk_consistent_lsn(); @@ -738,6 +745,8 @@ impl Timeline { walredo_mgr, + remote_client: remote_client.map(Arc::new), + // initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'. last_record_lsn: SeqWait::new(RecordLsn { last: disk_consistent_lsn, @@ -753,8 +762,6 @@ impl Timeline { metrics: TimelineMetrics::new(&tenant_id, &timeline_id), - upload_layers: AtomicBool::new(upload_layers), - flush_loop_started: Mutex::new(false), layer_flush_start_tx, @@ -884,7 +891,7 @@ impl Timeline { imgfilename, self.timeline_id, disk_consistent_lsn ); - rename_to_backup(direntry.path())?; + rename_to_backup(&direntry.path())?; continue; } @@ -908,7 +915,7 @@ impl Timeline { deltafilename, self.timeline_id, disk_consistent_lsn ); - rename_to_backup(direntry.path())?; + rename_to_backup(&direntry.path())?; continue; } @@ -945,6 +952,253 @@ impl Timeline { Ok(()) } + async fn download_missing( + &self, + index_part: &IndexPart, + remote_client: &RemoteTimelineClient, + mut local_filenames: HashSet, + first_save: bool, + ) -> anyhow::Result<(HashSet, TimelineMetadata)> { + let remote_metadata = index_part.parse_metadata().with_context(|| { + format!( + "Failed to parse metadata file from remote storage for tenant {}", + self.tenant_id + ) + })?; + + let remote_consistent_lsn = remote_metadata.disk_consistent_lsn(); + + let mut remote_filenames: HashSet = HashSet::new(); + for fname in index_part.timeline_layers.iter() { + remote_filenames.insert(fname.to_local_path(&PathBuf::from(""))); + } + + // Are there any local files that exist, with a size that doesn't match + // with the size stored in the remote index file? + local_filenames.retain(|path| { + let layer_metadata = index_part + .layer_metadata + .get(&RelativePath::from_filename(path)) + .map(LayerFileMetadata::from) + .unwrap_or(LayerFileMetadata::MISSING); + + if let Some(remote_size) = layer_metadata.file_size() { + let local_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id).join(&path); + match local_path.metadata() { + Ok(metadata) => { + let local_size = metadata.len(); + + if local_size != remote_size { + warn!("removing local file \"{}\" because it has unexpected length {}; length in remote index is {}", + path.display(), + local_size, + remote_size); + if let Err(err) = rename_to_backup(&local_path) { + error!("could not rename file \"{}\": {:?}", + local_path.display(), err); + } + false + } else { + true + } + } + Err(err) => { + error!("could not get size of local file \"{}\": {:?}", path.display(), err); + true + } + } + } else { + true + } + }); + + // Are we missing some files that are present in remote storage? + // Create RemoteLayers to represent them, and add to the layer map + // TODO Downloading many files this way is not efficient. + // Better to use FuturesUnordered. Maybe keep as is because: + // a) inplace download is a throw-away code, on-demand patch doesnt need that + // b) typical case now is that there is nothing to sync, this downloads a lot + // 1) if there was another pageserver that came and generated new files + // 2) during attach of a timeline with big history which we currently do not do + for path in remote_filenames.difference(&local_filenames) { + let fname = path.to_str().unwrap(); + info!("remote layer file {fname} does not exist locally"); + + let layer_metadata = index_part + .layer_metadata + .get(&RelativePath::from_filename(path)) + .map(LayerFileMetadata::from) + .unwrap_or(LayerFileMetadata::MISSING); + + if let Some(imgfilename) = ImageFileName::parse_str(fname) { + if imgfilename.lsn > remote_consistent_lsn { + warn!( + "found future image layer {} on timeline {} remote_consistent_lsn is {}", + imgfilename, self.timeline_id, remote_consistent_lsn + ); + continue; + } + + remote_client + .download_layer_file( + &RelativePath::from_local_path( + &self.conf.timeline_path(&self.timeline_id, &self.tenant_id), + path, + )?, + &layer_metadata, + ) + .await?; + + let image_layer = + ImageLayer::new(self.conf, self.timeline_id, self.tenant_id, &imgfilename); + + // FIXME: when to update physical size? + self.layers + .write() + .unwrap() + .insert_historic(Arc::new(image_layer)); + } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) { + // Create a DeltaLayer struct for each delta file. + // The end-LSN is exclusive, while disk_consistent_lsn is + // inclusive. For example, if disk_consistent_lsn is 100, it is + // OK for a delta layer to have end LSN 101, but if the end LSN + // is 102, then it might not have been fully flushed to disk + // before crash. + if deltafilename.lsn_range.end > remote_consistent_lsn + 1 { + warn!( + "found future delta layer {} on timeline {} remote_consistent_lsn is {}", + deltafilename, self.timeline_id, remote_consistent_lsn + ); + continue; + } + + let delta_layer = + DeltaLayer::new(self.conf, self.timeline_id, self.tenant_id, &deltafilename); + + // FIXME: when to update physical size? + self.layers + .write() + .unwrap() + .insert_historic(Arc::new(delta_layer)); + } else { + bail!("unexpected layer filename in remote storage: {}", fname); + } + } + + // Save the metadata file to local disk. + // Do this last, so that if we crash in-between, we won't think that the + // local state is valid. + // TODO we should not blindly overwrite local metadata with remote one + // consider the case when checkpoint came, we updated local metadata and started upload task + // but then pageserver crashes, then on start we'll load new metadata, and then reset it to the state of + // remote one. But current layermap will have layers from the old metadata which is inconsistent. + // if we treat remote as source of truth we need to completely syc with it, i e delete local files which are + // missing on the remote which will add extra work, wal for these layers needs to be reingested + // + // In other words we update local metadata only when remote one has greater disk_consistent_lsn + + // FIXME we should at least keep highest disk_consistent_lsn. Something else we need to merge? + + save_metadata( + self.conf, + self.timeline_id, + self.tenant_id, + &remote_metadata, + first_save, + )?; + + // now these are local only filenames + let local_only_filenames = local_filenames + .difference(&remote_filenames) + .cloned() + .collect(); + Ok((local_only_filenames, remote_metadata)) + } + + /// + /// This function will synchronize local data with what we have in remote storage. + /// 1. It will download missing layer files. + /// 2. It will update local metadata if remote one has greater `disk_consistent_lsn`. + /// 3. It will upload files that are missing on the remote + /// 4. It will update index file on the remote accordingly + /// TODO may be a bit cleaner to do things based on populated remote client, + /// and then do things based on its upload_queue.latest_files + /// + /// This is used during tenant attach. The layer map must have been loaded + /// with local filesystem contents already. + /// + /// The caller can provide IndexPart if it has it already. If it's None, + /// this function will download it. + /// + #[instrument(skip(self, index_part), fields(timeline_id = %self.timeline_id))] + pub async fn reconcile_with_remote( + &self, + local_metadata: TimelineMetadata, + index_part: Option<&IndexPart>, + first_save: bool, + ) -> anyhow::Result<()> { + info!("starting"); + let remote_client = self + .remote_client + .as_ref() + .ok_or_else(|| anyhow!("cannot download without remote storage"))?; + + // Build a map of local layers for quick lookups + let mut local_filenames: HashSet = HashSet::new(); + for layer in self.layers.read().unwrap().iter_historic_layers() { + local_filenames.insert(layer.filename()); + } + + // If the caller supplied an IndexPart, use it. Otherwise download it from remote storage. + // In case there is no such file on remote storage (which is possible if pageserver + // was interrupted before index file was uploaded). + // FIXME One way around it would be not considering timeline as initialized before upload finishes. + // So if there is no index file on the remote we consider all locally existing files as local only + // and schedule their upload + // FIXME looks messy, needs cleanup + let (local_only_filenames, up_to_date_metadata) = if let Some(index_part) = index_part { + let local_only_filenames = self + .download_missing(index_part, remote_client, local_filenames, first_save) + .await?; + remote_client.init_upload_queue(index_part)?; + local_only_filenames + } else { + match remote_client.download_index_file().await { + Ok(index_part) => { + let (local_only_filenames, up_to_date_metadata) = self + .download_missing(&index_part, remote_client, local_filenames, first_save) + .await?; + remote_client.init_upload_queue(&index_part)?; + (local_only_filenames, up_to_date_metadata) + } + Err(DownloadError::NotFound) => { + info!("no index file was found on the remote"); + remote_client.init_upload_queue_empty(&local_metadata)?; + (local_filenames, local_metadata) + } + Err(e) => return Err(anyhow::anyhow!(e)), + } + }; + + // TODO what to do with physical size? + + // Are there local files that don't exist remotely? Schedule uploads for them + for fname in &local_only_filenames { + let sz = fname.metadata()?.len(); + info!("scheduling {} for upload", fname.display()); + remote_client.schedule_layer_file_upload(fname, &LayerFileMetadata::new(sz))?; + } + if !local_only_filenames.is_empty() { + // FIXME this we should merge local and remote metadata, at least remote_consistent_lsn + // see comment in download_missing + remote_client.schedule_index_upload(Some(&up_to_date_metadata))?; + } + + info!("Done"); + + Ok(()) + } + pub(super) fn layer_removal_guard(&self) -> anyhow::Result> { self.layer_removal_cs .try_lock() @@ -1506,13 +1760,11 @@ impl Timeline { false, )?; - if self.can_upload_layers() { - storage_sync::schedule_layer_upload( - self.tenant_id, - self.timeline_id, - layer_paths_to_upload, - Some(metadata), - ); + if let Some(remote_client) = &self.remote_client { + for (path, layer_metadata) in layer_paths_to_upload { + remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; + } + remote_client.schedule_index_upload(Some(&metadata))?; } Ok(()) @@ -1971,32 +2223,26 @@ impl Timeline { // Now that we have reshuffled the data to set of new delta layers, we can // delete the old ones - let mut layer_paths_do_delete = HashSet::with_capacity(deltas_to_compact.len()); + let mut layer_paths_to_delete = Vec::with_capacity(deltas_to_compact.len()); drop(all_keys_iter); for l in deltas_to_compact { if let Some(path) = l.local_path() { self.metrics .current_physical_size_gauge .sub(path.metadata()?.len()); - layer_paths_do_delete.insert(path); + layer_paths_to_delete.push(path); } l.delete()?; layers.remove_historic(l); } drop(layers); - if self.can_upload_layers() { - storage_sync::schedule_layer_upload( - self.tenant_id, - self.timeline_id, - new_layer_paths, - None, - ); - storage_sync::schedule_layer_delete( - self.tenant_id, - self.timeline_id, - layer_paths_do_delete, - ); + // Also schedule the deletions in remote storage + if let Some(remote_client) = &self.remote_client { + // FIXME: This also uploads new index file. If + // flush_frozen_layer() is doing this at the same time, do + // we have a problem? + remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?; } Ok(()) @@ -2247,13 +2493,13 @@ impl Timeline { // Actually delete the layers from disk and remove them from the map. // (couldn't do this in the loop above, because you cannot modify a collection // while iterating it. BTreeMap::retain() would be another option) - let mut layer_paths_to_delete = HashSet::with_capacity(layers_to_remove.len()); + let mut layer_paths_to_delete = Vec::with_capacity(layers_to_remove.len()); for doomed_layer in layers_to_remove { if let Some(path) = doomed_layer.local_path() { self.metrics .current_physical_size_gauge .sub(path.metadata()?.len()); - layer_paths_to_delete.insert(path); + layer_paths_to_delete.push(path); } doomed_layer.delete()?; layers.remove_historic(doomed_layer); @@ -2269,12 +2515,8 @@ impl Timeline { fail_point!("after-timeline-gc-removed-layers"); } - if self.can_upload_layers() { - storage_sync::schedule_layer_delete( - self.tenant_id, - self.timeline_id, - layer_paths_to_delete, - ); + if let Some(remote_client) = &self.remote_client { + remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?; } result.elapsed = now.elapsed()?; @@ -2355,11 +2597,6 @@ impl Timeline { } } } - - fn can_upload_layers(&self) -> bool { - self.upload_layers.load(atomic::Ordering::Relaxed) - && self.current_state() != TimelineState::Broken - } } /// Helper function for get_reconstruct_data() to add the path of layers traversed @@ -2437,12 +2674,12 @@ impl<'a> TimelineWriter<'a> { /// Add a suffix to a layer file's name: .{num}.old /// Uses the first available num (starts at 0) -fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> { +fn rename_to_backup(path: &Path) -> anyhow::Result<()> { let filename = path .file_name() .ok_or_else(|| anyhow!("Path {} don't have a file name", path.display()))? .to_string_lossy(); - let mut new_path = path.clone(); + let mut new_path = path.to_owned(); for i in 0u32.. { new_path.set_file_name(format!("{}.{}.old", filename, i)); diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 3766bc5cb3..853447470c 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -1,7 +1,7 @@ //! This module acts as a switchboard to access different repositories managed by this //! page server. -use std::collections::{hash_map, HashMap}; +use std::collections::hash_map; use std::ffi::OsStr; use std::fs; use std::path::{Path, PathBuf}; @@ -12,14 +12,9 @@ use tracing::*; use remote_storage::GenericRemoteStorage; -use crate::config::{PageServerConf, METADATA_FILE_NAME, TIMELINE_UNINIT_MARK_SUFFIX}; -use crate::http::models::TenantInfo; -use crate::storage_sync::index::{LayerFileMetadata, RemoteIndex, RemoteTimelineIndex}; -use crate::storage_sync::{self, LocalTimelineInitStatus, SyncStartupData, TimelineLocalFiles}; +use crate::config::PageServerConf; use crate::task_mgr::{self, TaskKind}; -use crate::tenant::{ - ephemeral_file::is_ephemeral_file, metadata::TimelineMetadata, Tenant, TenantState, -}; +use crate::tenant::{Tenant, TenantState}; use crate::tenant_config::TenantConfOpt; use crate::walredo::PostgresRedoManager; use crate::TEMP_FILE_SUFFIX; @@ -59,153 +54,79 @@ mod tenants_state { pub fn init_tenant_mgr( conf: &'static PageServerConf, remote_storage: Option, -) -> anyhow::Result { +) -> anyhow::Result<()> { let _entered = info_span!("init_tenant_mgr").entered(); - let local_tenant_files = local_tenant_timeline_files(conf) - .context("Failed to collect local tenant timeline files")?; - - let (remote_index, tenants_to_attach) = if let Some(storage) = remote_storage { - let storage_config = conf - .remote_storage_config - .as_ref() - .expect("remote storage without config"); - let mut broken_tenants = HashMap::new(); - let mut ready_tenants = HashMap::new(); - for (tenant_id, tenant_attach_data) in local_tenant_files.into_iter() { - match tenant_attach_data { - TenantAttachData::Ready(t) => { - ready_tenants.insert(tenant_id, t); - } - TenantAttachData::Broken(e) => { - broken_tenants.insert(tenant_id, TenantAttachData::Broken(e)); + // Scan local filesystem for attached tenants + let mut number_of_tenants = 0; + let tenants_dir = conf.tenants_path(); + for dir_entry in std::fs::read_dir(&tenants_dir) + .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))? + { + match &dir_entry { + Ok(dir_entry) => { + let tenant_dir_path = dir_entry.path(); + if crate::is_temporary(&tenant_dir_path) { + info!( + "Found temporary tenant directory, removing: {}", + tenant_dir_path.display() + ); + if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) { + error!( + "Failed to remove temporary directory '{}': {:?}", + tenant_dir_path.display(), + e + ); + } + } else { + number_of_tenants += 1; + if let Err(e) = + load_local_tenant(conf, &tenant_dir_path, remote_storage.clone()) + { + error!( + "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}", + tenants_dir.display(), + dir_entry, + e + ); + } } } - } - let SyncStartupData { - remote_index, - local_timeline_init_statuses, - } = storage_sync::spawn_storage_sync_task( - conf, - ready_tenants, - storage, - storage_config.max_concurrent_syncs, - storage_config.max_sync_errors, - ) - .context("Failed to spawn the storage sync thread")?; - - let n = local_timeline_init_statuses.0.len(); - let mut synced_timelines = local_timeline_init_statuses.0.into_iter().fold( - HashMap::::with_capacity(n), - |mut new_values, (tenant_id, old_values)| { - let new_timeline_values = new_values - .entry(tenant_id) - .or_insert_with(|| TenantAttachData::Ready(HashMap::new())); - if let TenantAttachData::Ready(t) = new_timeline_values { - for (timeline_id, old_value) in old_values { - if let LocalTimelineInitStatus::LocallyComplete(metadata) = old_value { - t.insert(timeline_id, TimelineLocalFiles::ready(metadata)); - } - } - } - new_values - }, - ); - synced_timelines.extend(broken_tenants); - - (remote_index, synced_timelines) - } else { - info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled"); - (RemoteIndex::default(), local_tenant_files) - }; - attach_local_tenants(conf, &remote_index, tenants_to_attach); - - Ok(remote_index) -} - -/// Reads local files to load tenants and their timelines given into pageserver's memory. -/// Ignores other timelines that might be present for tenant, but were not passed as a parameter. -/// Attempts to load as many entites as possible: if a certain timeline fails during the load, the tenant is marked as "Broken", -/// and the load continues. -/// -/// For successful tenant attach, it first has to have a `timelines/` subdirectory and a tenant config file that's loaded into memory successfully. -/// If either of the conditions fails, the tenant will be added to memory with [`TenantState::Broken`] state, otherwise we start to load its timelines. -/// Alternatively, tenant is considered loaded successfully, if it's already in pageserver's memory (i.e. was loaded already before). -/// -/// Attach happens on startup and sucessful timeline downloads -/// (some subset of timeline files, always including its metadata, after which the new one needs to be registered). -pub fn attach_local_tenants( - conf: &'static PageServerConf, - remote_index: &RemoteIndex, - tenants_to_attach: HashMap, -) { - let _entered = info_span!("attach_local_tenants").entered(); - let number_of_tenants = tenants_to_attach.len(); - - for (tenant_id, local_timelines) in tenants_to_attach { - let mut tenants_accessor = tenants_state::write_tenants(); - let tenant = match tenants_accessor.entry(tenant_id) { - hash_map::Entry::Occupied(o) => { - info!("Tenant {tenant_id} was found in pageserver's memory"); - Arc::clone(o.get()) - } - hash_map::Entry::Vacant(v) => { - info!("Tenant {tenant_id} was not found in pageserver's memory, loading it"); - let tenant = Arc::new(Tenant::new( - conf, - TenantConfOpt::default(), - Arc::new(PostgresRedoManager::new(conf, tenant_id)), - tenant_id, - remote_index.clone(), - conf.remote_storage_config.is_some(), - )); - match local_timelines { - TenantAttachData::Broken(_) => { - tenant.set_state(TenantState::Broken); - } - TenantAttachData::Ready(_) => { - match Tenant::load_tenant_config(conf, tenant_id) { - Ok(tenant_conf) => { - tenant.update_tenant_config(tenant_conf); - tenant.activate(false); - } - Err(e) => { - error!("Failed to read config for tenant {tenant_id}, disabling tenant: {e:?}"); - tenant.set_state(TenantState::Broken); - } - }; - } - } - v.insert(Arc::clone(&tenant)); - tenant - } - }; - drop(tenants_accessor); - match local_timelines { - TenantAttachData::Broken(e) => warn!("{}", e), - TenantAttachData::Ready(ref timelines) => { - info!("Attaching {} timelines for {tenant_id}", timelines.len()); - debug!("Timelines to attach: {local_timelines:?}"); - let has_timelines = !timelines.is_empty(); - let timelines_to_attach = timelines - .iter() - .map(|(&k, v)| (k, v.metadata().to_owned())) - .collect(); - match tenant.init_attach_timelines(timelines_to_attach) { - Ok(()) => { - info!("successfully loaded local timelines for tenant {tenant_id}"); - tenant.activate(has_timelines); - } - Err(e) => { - error!("Failed to attach tenant timelines: {e:?}"); - tenant.set_state(TenantState::Broken); - } - } + Err(e) => { + // On error, print it, but continue with the other tenants. If we error out + // here, the pageserver startup fails altogether, causing outage for *all* + // tenants. That seems worse. + error!( + "Failed to list tenants dir entry {:?} in directory {}, reason: {:?}", + dir_entry, + tenants_dir.display(), + e, + ); } } } - info!("Processed {number_of_tenants} local tenants during attach") + info!("Processed {number_of_tenants} local tenants at startup"); + Ok(()) +} + +fn load_local_tenant( + conf: &'static PageServerConf, + tenant_path: &Path, + remote_storage: Option, +) -> anyhow::Result<()> { + let tenant_id = tenant_path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .parse::() + .context("Could not parse tenant id out of the tenant dir name")?; + + // Start loading the tenant into memory. It will initially be in Loading + // state. + let tenant = Tenant::spawn_load(conf, tenant_id, remote_storage)?; + tenants_state::write_tenants().insert(tenant_id, tenant); + Ok(()) } /// @@ -376,7 +297,7 @@ pub fn create_tenant( conf: &'static PageServerConf, tenant_conf: TenantConfOpt, tenant_id: TenantId, - remote_index: RemoteIndex, + remote_storage: Option<&GenericRemoteStorage>, ) -> anyhow::Result> { match tenants_state::write_tenants().entry(tenant_id) { hash_map::Entry::Occupied(_) => { @@ -386,15 +307,17 @@ pub fn create_tenant( hash_map::Entry::Vacant(v) => { let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); create_tenant_files(conf, tenant_conf, tenant_id)?; + // TODO review, created in active state immediately, is it correctly? let tenant = Arc::new(Tenant::new( + TenantState::Active { + background_jobs_running: true, + }, conf, tenant_conf, wal_redo_manager, tenant_id, - remote_index, - conf.remote_storage_config.is_some(), + remote_storage.cloned(), )); - tenant.activate(false); v.insert(tenant); Ok(Some(tenant_id)) } @@ -420,7 +343,10 @@ pub fn get_tenant(tenant_id: TenantId, active_only: bool) -> anyhow::Result Vec { +pub fn list_tenants() -> Vec<(TenantId, TenantState)> { tenants_state::read_tenants() .iter() - .map(|(id, tenant)| { - let has_in_progress_downloads = remote_index - .tenant_entry(id) - .map(|entry| entry.has_in_progress_downloads()); - - // TODO this is not correct when we might have remote storage sync disabled: - // we keep `RemoteTimelineIndex` in memory anyway for simplicity and this error message is printed still - if has_in_progress_downloads.is_none() { - error!("timeline is not found in remote index while it is present in the tenants registry") - } - - TenantInfo { - id: *id, - state: tenant.current_state(), - current_physical_size: None, - has_in_progress_downloads, - } - }) + .map(|(id, tenant)| (*id, tenant.current_state())) .collect() } -#[derive(Debug)] -pub enum TenantAttachData { - Ready(HashMap), - Broken(anyhow::Error), -} -/// Attempts to collect information about all tenant and timelines, existing on the local FS. -/// If finds any, deletes all temporary files and directories, created before. Also removes empty directories, -/// that may appear due to such removals. -/// Does not fail on particular timeline or tenant collection errors, rather logging them and ignoring the entities. -fn local_tenant_timeline_files( - config: &'static PageServerConf, -) -> anyhow::Result> { - let _entered = info_span!("local_tenant_timeline_files").entered(); - - let mut local_tenant_timeline_files = HashMap::new(); - let tenants_dir = config.tenants_path(); - for tenants_dir_entry in fs::read_dir(&tenants_dir) - .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))? - { - match &tenants_dir_entry { - Ok(tenants_dir_entry) => { - let tenant_dir_path = tenants_dir_entry.path(); - if is_temporary(&tenant_dir_path) { - info!( - "Found temporary tenant directory, removing: {}", - tenant_dir_path.display() - ); - if let Err(e) = fs::remove_dir_all(&tenant_dir_path) { - error!( - "Failed to remove temporary directory '{}': {:?}", - tenant_dir_path.display(), - e - ); - } - } else { - match collect_timelines_for_tenant(config, &tenant_dir_path) { - Ok((tenant_id, TenantAttachData::Broken(e))) => { - local_tenant_timeline_files.entry(tenant_id).or_insert(TenantAttachData::Broken(e)); - }, - Ok((tenant_id, TenantAttachData::Ready(collected_files))) => { - if collected_files.is_empty() { - match remove_if_empty(&tenant_dir_path) { - Ok(true) => info!("Removed empty tenant directory {}", tenant_dir_path.display()), - Ok(false) => { - // insert empty timeline entry: it has some non-temporary files inside that we cannot remove - // so make obvious for HTTP API callers, that something exists there and try to load the tenant - let _ = local_tenant_timeline_files.entry(tenant_id).or_insert_with(|| TenantAttachData::Ready(HashMap::new())); - }, - Err(e) => error!("Failed to remove empty tenant directory: {e:?}"), - } - } else { - match local_tenant_timeline_files.entry(tenant_id) { - hash_map::Entry::Vacant(entry) => { - entry.insert(TenantAttachData::Ready(collected_files)); - } - hash_map::Entry::Occupied(entry) =>{ - if let TenantAttachData::Ready(old_timelines) = entry.into_mut() { - old_timelines.extend(collected_files); - } - }, - } - } - }, - Err(e) => error!( - "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}", - tenants_dir.display(), - tenants_dir_entry, - e - ), - } +/// Execute Attach mgmt API command. +/// +/// Downloading all the tenant data is performed in the background, this merely +/// spawns the background task and returns quickly. +pub async fn attach_tenant( + conf: &'static PageServerConf, + tenant_id: TenantId, + remote_storage: &GenericRemoteStorage, +) -> anyhow::Result<()> { + match tenants_state::write_tenants().entry(tenant_id) { + hash_map::Entry::Occupied(e) => { + // Cannot attach a tenant that already exists. The error message depends on + // the state it's in. + match e.get().current_state() { + TenantState::Attaching => { + anyhow::bail!("tenant {tenant_id} attach is already in progress") + } + current_state => { + anyhow::bail!("tenant already exists, current state: {current_state:?}") } } - Err(e) => error!( - "Failed to list tenants dir entry {:?} in directory {}, reason: {:?}", - tenants_dir_entry, - tenants_dir.display(), - e - ), + } + hash_map::Entry::Vacant(v) => { + let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage)?; + v.insert(tenant); + Ok(()) } } - - info!( - "Collected files for {} tenants", - local_tenant_timeline_files.len(), - ); - Ok(local_tenant_timeline_files) -} - -fn remove_if_empty(tenant_dir_path: &Path) -> anyhow::Result { - let directory_is_empty = tenant_dir_path - .read_dir() - .with_context(|| { - format!( - "Failed to read directory '{}' contents", - tenant_dir_path.display() - ) - })? - .next() - .is_none(); - - if directory_is_empty { - fs::remove_dir_all(&tenant_dir_path).with_context(|| { - format!( - "Failed to remove empty directory '{}'", - tenant_dir_path.display(), - ) - })?; - - Ok(true) - } else { - Ok(false) - } -} - -fn is_temporary(path: &Path) -> bool { - match path.file_name() { - Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX), - None => false, - } -} - -fn is_uninit_mark(path: &Path) -> bool { - match path.file_name() { - Some(name) => name - .to_string_lossy() - .ends_with(TIMELINE_UNINIT_MARK_SUFFIX), - None => false, - } -} - -fn collect_timelines_for_tenant( - config: &'static PageServerConf, - tenant_path: &Path, -) -> anyhow::Result<(TenantId, TenantAttachData)> { - let tenant_id = tenant_path - .file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .parse::() - .context("Could not parse tenant id out of the tenant dir name")?; - let timelines_dir = config.timelines_path(&tenant_id); - - if !timelines_dir.as_path().is_dir() { - return Ok(( - tenant_id, - TenantAttachData::Broken(anyhow::anyhow!( - "Tenant {} has no timelines directory at {}", - tenant_id, - timelines_dir.display() - )), - )); - } - - let mut tenant_timelines = HashMap::new(); - for timelines_dir_entry in fs::read_dir(&timelines_dir) - .with_context(|| format!("Failed to list timelines dir entry for tenant {tenant_id}"))? - { - match timelines_dir_entry { - Ok(timelines_dir_entry) => { - let timeline_dir = timelines_dir_entry.path(); - if is_temporary(&timeline_dir) { - info!( - "Found temporary timeline directory, removing: {}", - timeline_dir.display() - ); - if let Err(e) = fs::remove_dir_all(&timeline_dir) { - error!( - "Failed to remove temporary directory '{}': {:?}", - timeline_dir.display(), - e - ); - } - } else if is_uninit_mark(&timeline_dir) { - let timeline_uninit_mark_file = &timeline_dir; - info!( - "Found an uninit mark file {}, removing the timeline and its uninit mark", - timeline_uninit_mark_file.display() - ); - let timeline_id = timeline_uninit_mark_file - .file_stem() - .and_then(OsStr::to_str) - .unwrap_or_default() - .parse::() - .with_context(|| { - format!( - "Could not parse timeline id out of the timeline uninit mark name {}", - timeline_uninit_mark_file.display() - ) - })?; - let timeline_dir = config.timeline_path(&timeline_id, &tenant_id); - if let Err(e) = - remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file) - { - error!("Failed to clean up uninit marked timeline: {e:?}"); - } - } else { - let timeline_id = timeline_dir - .file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .parse::() - .with_context(|| { - format!( - "Could not parse timeline id out of the timeline dir name {}", - timeline_dir.display() - ) - })?; - let timeline_uninit_mark_file = - config.timeline_uninit_mark_file_path(tenant_id, timeline_id); - if timeline_uninit_mark_file.exists() { - info!("Found an uninit mark file for timeline {tenant_id}/{timeline_id}, removing the timeline and its uninit mark"); - if let Err(e) = remove_timeline_and_uninit_mark( - &timeline_dir, - &timeline_uninit_mark_file, - ) { - error!("Failed to clean up uninit marked timeline: {e:?}"); - } - } else { - match collect_timeline_files(&timeline_dir) { - Ok((metadata, timeline_files)) => { - tenant_timelines.insert( - timeline_id, - TimelineLocalFiles::collected(metadata, timeline_files), - ); - } - Err(e) => { - error!( - "Failed to process timeline dir contents at '{}', reason: {:?}", - timeline_dir.display(), - e - ); - match remove_if_empty(&timeline_dir) { - Ok(true) => info!( - "Removed empty timeline directory {}", - timeline_dir.display() - ), - Ok(false) => (), - Err(e) => { - error!("Failed to remove empty timeline directory: {e:?}") - } - } - } - } - } - } - } - Err(e) => { - error!("Failed to list timelines for entry tenant {tenant_id}, reason: {e:?}") - } - } - } - - if tenant_timelines.is_empty() { - // this is normal, we've removed all broken, empty and temporary timeline dirs - // but should allow the tenant to stay functional and allow creating new timelines - // on a restart, we require tenants to have the timelines dir, so leave it on disk - debug!("Tenant {tenant_id} has no timelines loaded"); - } - - Ok((tenant_id, TenantAttachData::Ready(tenant_timelines))) -} - -fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> { - fs::remove_dir_all(&timeline_dir) - .or_else(|e| { - if e.kind() == std::io::ErrorKind::NotFound { - // we can leave the uninit mark without a timeline dir, - // just remove the mark then - Ok(()) - } else { - Err(e) - } - }) - .with_context(|| { - format!( - "Failed to remove unit marked timeline directory {}", - timeline_dir.display() - ) - })?; - fs::remove_file(&uninit_mark).with_context(|| { - format!( - "Failed to remove timeline uninit mark file {}", - uninit_mark.display() - ) - })?; - - Ok(()) -} - -// discover timeline files and extract timeline metadata -// NOTE: ephemeral files are excluded from the list -fn collect_timeline_files( - timeline_dir: &Path, -) -> anyhow::Result<(TimelineMetadata, HashMap)> { - let mut timeline_files = HashMap::new(); - let mut timeline_metadata_path = None; - - let timeline_dir_entries = - fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?; - for entry in timeline_dir_entries { - let entry_path = entry.context("Failed to list timeline dir entry")?.path(); - let metadata = entry_path.metadata()?; - - if metadata.is_file() { - if entry_path.file_name().and_then(OsStr::to_str) == Some(METADATA_FILE_NAME) { - timeline_metadata_path = Some(entry_path); - } else if is_ephemeral_file(&entry_path.file_name().unwrap().to_string_lossy()) { - debug!("skipping ephemeral file {}", entry_path.display()); - continue; - } else if is_temporary(&entry_path) { - info!("removing temp timeline file at {}", entry_path.display()); - fs::remove_file(&entry_path).with_context(|| { - format!( - "failed to remove temp download file at {}", - entry_path.display() - ) - })?; - } else { - let layer_metadata = LayerFileMetadata::new(metadata.len()); - timeline_files.insert(entry_path, layer_metadata); - } - } - } - - // FIXME (rodionov) if attach call succeeded, and then pageserver is restarted before download is completed - // then attach is lost. There would be no retries for that, - // initial collect will fail because there is no metadata. - // We either need to start download if we see empty dir after restart or attach caller should - // be aware of that and retry attach if awaits_download for timeline switched from true to false - // but timelinne didn't appear locally. - // Check what happens with remote index in that case. - let timeline_metadata_path = match timeline_metadata_path { - Some(path) => path, - None => anyhow::bail!("No metadata file found in the timeline directory"), - }; - let metadata = TimelineMetadata::from_bytes( - &fs::read(&timeline_metadata_path).context("Failed to read timeline metadata file")?, - ) - .context("Failed to parse timeline metadata file bytes")?; - - anyhow::ensure!( - metadata.ancestor_timeline().is_some() || !timeline_files.is_empty(), - "Timeline has no ancestor and no layer files" - ); - - Ok((metadata, timeline_files)) } diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 8c81ed824b..bd7c08a31f 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1090,9 +1090,9 @@ mod tests { Ok(walingest) } - #[test] - fn test_relsize() -> Result<()> { - let tenant = TenantHarness::create("test_relsize")?.load(); + #[tokio::test] + async fn test_relsize() -> Result<()> { + let tenant = TenantHarness::create("test_relsize")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&*tline)?; @@ -1218,9 +1218,9 @@ mod tests { // Test what happens if we dropped a relation // and then created it again within the same layer. - #[test] - fn test_drop_extend() -> Result<()> { - let tenant = TenantHarness::create("test_drop_extend")?.load(); + #[tokio::test] + async fn test_drop_extend() -> Result<()> { + let tenant = TenantHarness::create("test_drop_extend")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&*tline)?; @@ -1258,9 +1258,9 @@ mod tests { // Test what happens if we truncated a relation // so that one of its segments was dropped // and then extended it again within the same layer. - #[test] - fn test_truncate_extend() -> Result<()> { - let tenant = TenantHarness::create("test_truncate_extend")?.load(); + #[tokio::test] + async fn test_truncate_extend() -> Result<()> { + let tenant = TenantHarness::create("test_truncate_extend")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&*tline)?; @@ -1346,9 +1346,9 @@ mod tests { /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's /// split into multiple 1 GB segments in Postgres. - #[test] - fn test_large_rel() -> Result<()> { - let tenant = TenantHarness::create("test_large_rel")?.load(); + #[tokio::test] + async fn test_large_rel() -> Result<()> { + let tenant = TenantHarness::create("test_large_rel")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&*tline)?; diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index d527e521e0..0654e3e22b 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -857,10 +857,10 @@ mod tests { use super::*; use crate::tenant::harness::{TenantHarness, TIMELINE_ID}; - #[test] - fn no_connection_no_candidate() -> anyhow::Result<()> { + #[tokio::test] + async fn no_connection_no_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("no_connection_no_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); let lagging_wal_timeout = chrono::Duration::from_std(state.lagging_wal_timeout)?; @@ -951,7 +951,7 @@ mod tests { #[tokio::test] async fn connection_no_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("connection_no_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); let connected_sk_id = NodeId(0); @@ -1045,10 +1045,10 @@ mod tests { Ok(()) } - #[test] - fn no_connection_candidate() -> anyhow::Result<()> { + #[tokio::test] + async fn no_connection_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("no_connection_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); state.wal_connection = None; @@ -1161,7 +1161,7 @@ mod tests { #[tokio::test] async fn candidate_with_many_connection_failures() -> anyhow::Result<()> { let harness = TenantHarness::create("candidate_with_many_connection_failures")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); let current_lsn = Lsn(100_000).align(); @@ -1229,7 +1229,7 @@ mod tests { #[tokio::test] async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1320,7 +1320,7 @@ mod tests { #[tokio::test] async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1394,7 +1394,7 @@ mod tests { #[tokio::test] async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?; - let mut state = dummy_state(&harness); + let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let new_lsn = Lsn(100_100).align(); let now = Utc::now().naive_utc(); @@ -1472,7 +1472,7 @@ mod tests { const DUMMY_SAFEKEEPER_CONNSTR: &str = "safekeeper_connstr"; - fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState { + async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState { WalreceiverState { id: TenantTimelineId { tenant_id: harness.tenant_id, @@ -1480,6 +1480,7 @@ mod tests { }, timeline: harness .load() + .await .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION) .expect("Failed to create an empty timeline for dummy wal connection manager") .initialize() diff --git a/pageserver/src/walreceiver/walreceiver_connection.rs b/pageserver/src/walreceiver/walreceiver_connection.rs index 0070834288..f3aabff973 100644 --- a/pageserver/src/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/walreceiver/walreceiver_connection.rs @@ -26,13 +26,12 @@ use crate::{ task_mgr::TaskKind, task_mgr::WALRECEIVER_RUNTIME, tenant::{Timeline, WalReceiverInfo}, - tenant_mgr, walingest::WalIngest, walrecord::DecodedWALRecord, }; use postgres_ffi::waldecoder::WalStreamDecoder; use pq_proto::ReplicationFeedback; -use utils::{id::TenantTimelineId, lsn::Lsn}; +use utils::lsn::Lsn; /// Status of the connection. #[derive(Debug, Clone)] @@ -139,10 +138,6 @@ pub async fn handle_walreceiver_connection( return Ok(()); } - let tenant_id = timeline.tenant_id; - let timeline_id = timeline.timeline_id; - let tenant = tenant_mgr::get_tenant(tenant_id, true)?; - // // Start streaming the WAL, from where we left off previously. // @@ -291,19 +286,8 @@ pub async fn handle_walreceiver_connection( })?; if let Some(last_lsn) = status_update { - let remote_index = tenant.get_remote_index(); - let timeline_remote_consistent_lsn = remote_index - .read() - .await - // here we either do not have this timeline in remote index - // or there were no checkpoints for it yet - .timeline_entry(&TenantTimelineId { - tenant_id, - timeline_id, - }) - .map(|remote_timeline| remote_timeline.metadata.disk_consistent_lsn()) - // no checkpoint was uploaded - .unwrap_or(Lsn(0)); + let timeline_remote_consistent_lsn = + timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0)); // The last LSN we processed. It is not guaranteed to survive pageserver crash. let write_lsn = u64::from(last_lsn); diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index 5abf0967f2..79a49c59ec 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -77,7 +77,9 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): # But all others are broken # First timeline would not get loaded into pageserver due to corrupt metadata file - with pytest.raises(Exception, match=f"Timeline {tenant1}/{timeline1} was not found") as err: + with pytest.raises( + Exception, match=f"Tenant {tenant1} is not active. Current state: Broken" + ) as err: pg1.start() log.info( f"As expected, compute startup failed eagerly for timeline with corrupt metadata: {err}" @@ -86,7 +88,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): # Second timeline has no ancestors, only the metadata file and no layer files # We don't have the remote storage enabled, which means timeline is in an incorrect state, # it's not loaded at all - with pytest.raises(Exception, match=f"Timeline {tenant2}/{timeline2} was not found") as err: + with pytest.raises(Exception, match=".*extracting base backup failed") as err: pg2.start() log.info(f"As expected, compute startup failed for timeline with missing layers: {err}")