diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index a3772265c0..03280586f8 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -387,10 +387,20 @@ COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ ARG PG_VERSION ENV PATH "/usr/local/pgsql/bin:$PATH" -RUN apt-get update && \ +RUN case "${PG_VERSION}" in \ + "v14" | "v15") \ + export TIMESCALEDB_VERSION=2.10.1 \ + export TIMESCALEDB_CHECKSUM=6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 \ + ;; \ + *) \ + export TIMESCALEDB_VERSION=2.13.0 \ + export TIMESCALEDB_CHECKSUM=584a351c7775f0e067eaa0e7277ea88cab9077cc4c455cbbf09a5d9723dce95d \ + ;; \ + esac && \ + apt-get update && \ apt-get install -y cmake && \ - wget https://github.com/timescale/timescaledb/archive/refs/tags/2.13.0.tar.gz -O timescaledb.tar.gz && \ - echo "584a351c7775f0e067eaa0e7277ea88cab9077cc4c455cbbf09a5d9723dce95d timescaledb.tar.gz" | sha256sum --check && \ + wget https://github.com/timescale/timescaledb/archive/refs/tags/${TIMESCALEDB_VERSION}.tar.gz -O timescaledb.tar.gz && \ + echo "${TIMESCALEDB_CHECKSUM} timescaledb.tar.gz" | sha256sum --check && \ mkdir timescaledb-src && cd timescaledb-src && tar xvzf ../timescaledb.tar.gz --strip-components=1 -C . && \ ./bootstrap -DSEND_TELEMETRY_DEFAULT:BOOL=OFF -DUSE_TELEMETRY:BOOL=OFF -DAPACHE_ONLY:BOOL=ON -DCMAKE_BUILD_TYPE=Release && \ cd build && \ diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs index 7bcc0ee4c6..f8f9449d86 100644 --- a/libs/pageserver_api/src/shard.rs +++ b/libs/pageserver_api/src/shard.rs @@ -73,19 +73,28 @@ impl TenantShardId { ) } - pub fn shard_slug(&self) -> String { - format!("{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + pub fn shard_slug(&self) -> impl std::fmt::Display + '_ { + ShardSlug(self) + } +} + +/// Formatting helper +struct ShardSlug<'a>(&'a TenantShardId); + +impl<'a> std::fmt::Display for ShardSlug<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:02x}{:02x}", + self.0.shard_number.0, self.0.shard_count.0 + ) } } impl std::fmt::Display for TenantShardId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if self.shard_count != ShardCount(0) { - write!( - f, - "{}-{:02x}{:02x}", - self.tenant_id, self.shard_number.0, self.shard_count.0 - ) + write!(f, "{}-{}", self.tenant_id, self.shard_slug()) } else { // Legacy case (shard_count == 0) -- format as just the tenant id. Note that this // is distinct from the normal single shard case (shard count == 1). diff --git a/libs/utils/src/simple_rcu.rs b/libs/utils/src/simple_rcu.rs index 177a839d75..dc4a599111 100644 --- a/libs/utils/src/simple_rcu.rs +++ b/libs/utils/src/simple_rcu.rs @@ -1,10 +1,10 @@ //! //! RCU stands for Read-Copy-Update. It's a synchronization mechanism somewhat //! similar to a lock, but it allows readers to "hold on" to an old value of RCU -//! without blocking writers, and allows writing a new values without blocking -//! readers. When you update the new value, the new value is immediately visible +//! without blocking writers, and allows writing a new value without blocking +//! readers. When you update the value, the new value is immediately visible //! to new readers, but the update waits until all existing readers have -//! finishe, so that no one sees the old value anymore. +//! finished, so that on return, no one sees the old value anymore. //! //! This implementation isn't wait-free; it uses an RwLock that is held for a //! short duration when the value is read or updated. @@ -26,6 +26,7 @@ //! Increment the value by one, and wait for old readers to finish: //! //! ``` +//! # async fn dox() { //! # let rcu = utils::simple_rcu::Rcu::new(1); //! let write_guard = rcu.lock_for_write(); //! @@ -36,15 +37,17 @@ //! //! // Concurrent reads and writes are now possible again. Wait for all the readers //! // that still observe the old value to finish. -//! waitlist.wait(); +//! waitlist.wait().await; +//! # } //! ``` //! #![warn(missing_docs)] use std::ops::Deref; -use std::sync::mpsc::{sync_channel, Receiver, SyncSender}; use std::sync::{Arc, Weak}; -use std::sync::{Mutex, RwLock, RwLockWriteGuard}; +use std::sync::{RwLock, RwLockWriteGuard}; + +use tokio::sync::watch; /// /// Rcu allows multiple readers to read and hold onto a value without blocking @@ -68,22 +71,21 @@ struct RcuCell { value: V, /// A dummy channel. We never send anything to this channel. The point is - /// that when the RcuCell is dropped, any cloned Senders will be notified + /// that when the RcuCell is dropped, any subscribed Receivers will be notified /// that the channel is closed. Updaters can use this to wait out until the /// RcuCell has been dropped, i.e. until the old value is no longer in use. /// - /// We never do anything with the receiver, we just need to hold onto it so - /// that the Senders will be notified when it's dropped. But because it's - /// not Sync, we need a Mutex on it. - watch: (SyncSender<()>, Mutex>), + /// We never send anything to this, we just need to hold onto it so that the + /// Receivers will be notified when it's dropped. + watch: watch::Sender<()>, } impl RcuCell { fn new(value: V) -> Self { - let (watch_sender, watch_receiver) = sync_channel(0); + let (watch_sender, _) = watch::channel(()); RcuCell { value, - watch: (watch_sender, Mutex::new(watch_receiver)), + watch: watch_sender, } } } @@ -141,10 +143,10 @@ impl Deref for RcuReadGuard { /// /// Write guard returned by `write` /// -/// NB: Holding this guard blocks all concurrent `read` and `write` calls, so -/// it should only be held for a short duration! +/// NB: Holding this guard blocks all concurrent `read` and `write` calls, so it should only be +/// held for a short duration! /// -/// Calling `store` consumes the guard, making new reads and new writes possible +/// Calling [`Self::store_and_unlock`] consumes the guard, making new reads and new writes possible /// again. /// pub struct RcuWriteGuard<'a, V> { @@ -179,7 +181,7 @@ impl<'a, V> RcuWriteGuard<'a, V> { // the watches for any that do. self.inner.old_cells.retain(|weak| { if let Some(cell) = weak.upgrade() { - watches.push(cell.watch.0.clone()); + watches.push(cell.watch.subscribe()); true } else { false @@ -193,20 +195,20 @@ impl<'a, V> RcuWriteGuard<'a, V> { /// /// List of readers who can still see old values. /// -pub struct RcuWaitList(Vec>); +pub struct RcuWaitList(Vec>); impl RcuWaitList { /// /// Wait for old readers to finish. /// - pub fn wait(mut self) { + pub async fn wait(mut self) { // after all the old_cells are no longer in use, we're done for w in self.0.iter_mut() { // This will block until the Receiver is closed. That happens when // the RcuCell is dropped. #[allow(clippy::single_match)] - match w.send(()) { - Ok(_) => panic!("send() unexpectedly succeeded on dummy channel"), + match w.changed().await { + Ok(_) => panic!("changed() unexpectedly succeeded on dummy channel"), Err(_) => { // closed, which means that the cell has been dropped, and // its value is no longer in use @@ -220,11 +222,10 @@ impl RcuWaitList { mod tests { use super::*; use std::sync::{Arc, Mutex}; - use std::thread::{sleep, spawn}; use std::time::Duration; - #[test] - fn two_writers() { + #[tokio::test] + async fn two_writers() { let rcu = Rcu::new(1); let read1 = rcu.read(); @@ -248,33 +249,35 @@ mod tests { assert_eq!(*read1, 1); let log = Arc::new(Mutex::new(Vec::new())); - // Wait for the old readers to finish in separate threads. + // Wait for the old readers to finish in separate tasks. let log_clone = Arc::clone(&log); - let thread2 = spawn(move || { - wait2.wait(); + let task2 = tokio::spawn(async move { + wait2.wait().await; log_clone.lock().unwrap().push("wait2 done"); }); let log_clone = Arc::clone(&log); - let thread3 = spawn(move || { - wait3.wait(); + let task3 = tokio::spawn(async move { + wait3.wait().await; log_clone.lock().unwrap().push("wait3 done"); }); // without this sleep the test can pass on accident if the writer is slow - sleep(Duration::from_millis(500)); + tokio::time::sleep(Duration::from_millis(100)).await; // Release first reader. This allows first write to finish, but calling - // wait() on the second one would still block. + // wait() on the 'task3' would still block. log.lock().unwrap().push("dropping read1"); drop(read1); - thread2.join().unwrap(); + task2.await.unwrap(); - sleep(Duration::from_millis(500)); + assert!(!task3.is_finished()); + + tokio::time::sleep(Duration::from_millis(100)).await; // Release second reader, and finish second writer. log.lock().unwrap().push("dropping read2"); drop(read2); - thread3.join().unwrap(); + task3.await.unwrap(); assert_eq!( log.lock().unwrap().as_slice(), diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 14b667eeba..9e41d912c2 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -844,7 +844,7 @@ async fn tenant_delete_handler( mgr::delete_tenant(state.conf, state.remote_storage.clone(), tenant_shard_id) .instrument(info_span!("tenant_delete_handler", tenant_id = %tenant_shard_id.tenant_id, - shard = tenant_shard_id.shard_slug() + shard = %tenant_shard_id.shard_slug() )) .await?; @@ -1193,7 +1193,7 @@ async fn put_tenant_location_config_handler( mgr::detach_tenant(conf, tenant_shard_id, true, &state.deletion_queue_client) .instrument(info_span!("tenant_detach", tenant_id = %tenant_shard_id.tenant_id, - shard = tenant_shard_id.shard_slug() + shard = %tenant_shard_id.shard_slug() )) .await { diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 6e311041ba..b684c4fa90 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -2094,6 +2094,8 @@ pub fn preinitialize_metrics() { // Tenant manager stats Lazy::force(&TENANT_MANAGER); + Lazy::force(&crate::tenant::storage_layer::layer::LAYER_IMPL_METRICS); + // countervecs [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT] .into_iter() diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 9956e761e6..d5ca7f7382 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -67,9 +67,9 @@ use crate::trace::Tracer; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; -// How long we may block waiting for a [`TenantSlot::InProgress`]` and/or a [`Tenant`] which +// How long we may wait for a [`TenantSlot::InProgress`]` and/or a [`Tenant`] which // is not yet in state [`TenantState::Active`]. -const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000); +const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(30000); /// Read the end of a tar archive. /// diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 12643cf61d..8466fe7fca 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1937,7 +1937,7 @@ fn tenant_map_acquire_slot_impl( METRICS.tenant_slot_writes.inc(); let mut locked = tenants.write().unwrap(); - let span = tracing::info_span!("acquire_slot", tenant_id=%tenant_shard_id.tenant_id, shard=tenant_shard_id.shard_slug()); + let span = tracing::info_span!("acquire_slot", tenant_id=%tenant_shard_id.tenant_id, shard = %tenant_shard_id.shard_slug()); let _guard = span.enter(); let m = match &mut *locked { diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 693bb1e751..1c0d803695 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -4,7 +4,7 @@ pub mod delta_layer; mod filename; pub mod image_layer; mod inmemory_layer; -mod layer; +pub(crate) mod layer; mod layer_desc; use crate::context::{AccessStatsBehavior, RequestContext}; diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index fa617f1a7d..de29ad4dcd 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -214,8 +214,8 @@ impl Layer { /// /// [gc]: [`RemoteTimelineClient::schedule_gc_update`] /// [compaction]: [`RemoteTimelineClient::schedule_compaction_update`] - pub(crate) fn garbage_collect_on_drop(&self) { - self.0.garbage_collect_on_drop(); + pub(crate) fn delete_on_drop(&self) { + self.0.delete_on_drop(); } /// Return data needed to reconstruct given page at LSN. @@ -327,10 +327,10 @@ impl Layer { Ok(()) } - /// Waits until this layer has been dropped (and if needed, local garbage collection and remote + /// Waits until this layer has been dropped (and if needed, local file deletion and remote /// deletion scheduling has completed). /// - /// Does not start garbage collection, use [`Self::garbage_collect_on_drop`] for that + /// Does not start local deletion, use [`Self::delete_on_drop`] for that /// separatedly. #[cfg(feature = "testing")] pub(crate) fn wait_drop(&self) -> impl std::future::Future + 'static { @@ -415,8 +415,8 @@ struct LayerInner { /// Initialization and deinitialization are done while holding a permit. inner: heavier_once_cell::OnceCell, - /// Do we want to garbage collect this when `LayerInner` is dropped - wanted_garbage_collected: AtomicBool, + /// Do we want to delete locally and remotely this when `LayerInner` is dropped + wanted_deleted: AtomicBool, /// Do we want to evict this layer as soon as possible? After being set to `true`, all accesses /// will try to downgrade [`ResidentOrWantedEvicted`], which will eventually trigger @@ -430,10 +430,6 @@ struct LayerInner { version: AtomicUsize, /// Allow subscribing to when the layer actually gets evicted. - /// - /// If in future we need to implement "wait until layer instances are gone and done", carrying - /// this over to the gc spawn_blocking from LayerInner::drop will do the trick, and adding a - /// method for "wait_gc" which will wait to this being closed. status: tokio::sync::broadcast::Sender, /// Counter for exponential backoff with the download @@ -475,7 +471,7 @@ enum Status { impl Drop for LayerInner { fn drop(&mut self) { - if !*self.wanted_garbage_collected.get_mut() { + if !*self.wanted_deleted.get_mut() { // should we try to evict if the last wish was for eviction? // feels like there's some hazard of overcrowding near shutdown near by, but we don't // run drops during shutdown (yet) @@ -498,7 +494,7 @@ impl Drop for LayerInner { }; // If timeline is alive, we can construct a span with IDs for this function. - let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %timeline.tenant_shard_id.tenant_id, shard_id=%timeline.tenant_shard_id.shard_slug(), timeline_id = %timeline.timeline_id); + let span = tracing::info_span!(parent: None, "layer_delete", tenant_id = %timeline.tenant_shard_id.tenant_id, shard_id=%timeline.tenant_shard_id.shard_slug(), timeline_id = %timeline.timeline_id); let path = self.build_local_path(&timeline.tenant_shard_id, &timeline.timeline_id); let file_name = self.layer_desc().filename(); @@ -525,8 +521,8 @@ impl Drop for LayerInner { false } Err(e) => { - tracing::error!("failed to remove garbage collected layer: {e}"); - LAYER_IMPL_METRICS.inc_gc_removes_failed(); + tracing::error!("failed to remove wanted deleted layer: {e}"); + LAYER_IMPL_METRICS.inc_delete_removes_failed(); false } }; @@ -548,15 +544,15 @@ impl Drop for LayerInner { } else { tracing::warn!("scheduling deletion on drop failed: {e:#}"); } - LAYER_IMPL_METRICS.inc_gcs_failed(GcFailed::DeleteSchedulingFailed); + LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::DeleteSchedulingFailed); } else { - LAYER_IMPL_METRICS.inc_completed_gcs(); + LAYER_IMPL_METRICS.inc_completed_deletes(); } } } else { // no need to nag that timeline is gone: under normal situation on // task_mgr::remove_tenant_from_memory the timeline is gone before we get dropped. - LAYER_IMPL_METRICS.inc_gcs_failed(GcFailed::TimelineGone); + LAYER_IMPL_METRICS.inc_deletes_failed(DeleteFailed::TimelineGone); } }); } @@ -586,7 +582,7 @@ impl LayerInner { timeline: Arc::downgrade(timeline), have_remote_client: timeline.remote_client.is_some(), access_stats, - wanted_garbage_collected: AtomicBool::new(false), + wanted_deleted: AtomicBool::new(false), wanted_evicted: AtomicBool::new(false), inner, version: AtomicUsize::new(version), @@ -620,16 +616,13 @@ impl LayerInner { .join(self.desc.filename().to_string()) } - fn garbage_collect_on_drop(&self) { - let res = self.wanted_garbage_collected.compare_exchange( - false, - true, - Ordering::Release, - Ordering::Relaxed, - ); + fn delete_on_drop(&self) { + let res = + self.wanted_deleted + .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed); if res.is_ok() { - LAYER_IMPL_METRICS.inc_started_gcs(); + LAYER_IMPL_METRICS.inc_started_deletes(); } } @@ -697,6 +690,10 @@ impl LayerInner { // disable any scheduled but not yet running eviction deletions for this let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed); + // count cancellations, which currently remain largely unexpected + let init_cancelled = + scopeguard::guard((), |_| LAYER_IMPL_METRICS.inc_init_cancelled()); + // no need to make the evict_and_wait wait for the actual download to complete drop(self.status.send(Status::Downloaded)); @@ -705,6 +702,8 @@ impl LayerInner { .upgrade() .ok_or_else(|| DownloadError::TimelineShutdown)?; + // FIXME: grab a gate + let can_ever_evict = timeline.remote_client.as_ref().is_some(); // check if we really need to be downloaded; could have been already downloaded by a @@ -765,6 +764,8 @@ impl LayerInner { tracing::info!(waiters, "completing the on-demand download for other tasks"); } + scopeguard::ScopeGuard::into_inner(init_cancelled); + Ok((ResidentOrWantedEvicted::Resident(res), permit)) }; @@ -893,14 +894,13 @@ impl LayerInner { match res { (Ok(()), _) => { // our caller is cancellation safe so this is fine; if someone - // else requests the layer, they'll find it already downloaded - // or redownload. + // else requests the layer, they'll find it already downloaded. // - // however, could be that we should consider marking the layer - // for eviction? alas, cannot: because only DownloadedLayer - // will handle that. - tracing::info!("layer file download completed after requester had cancelled"); - LAYER_IMPL_METRICS.inc_download_completed_without_requester(); + // See counter [`LayerImplMetrics::inc_init_needed_no_download`] + // + // FIXME(#6028): however, could be that we should consider marking the + // layer for eviction? alas, cannot: because only DownloadedLayer will + // handle that. }, (Err(e), _) => { // our caller is cancellation safe, but we might be racing with @@ -1028,12 +1028,15 @@ impl LayerInner { /// `DownloadedLayer` is being dropped, so it calls this method. fn on_downloaded_layer_drop(self: Arc, version: usize) { - let gc = self.wanted_garbage_collected.load(Ordering::Acquire); + let delete = self.wanted_deleted.load(Ordering::Acquire); let evict = self.wanted_evicted.load(Ordering::Acquire); let can_evict = self.have_remote_client; - if gc { - // do nothing now, only in LayerInner::drop + if delete { + // do nothing now, only in LayerInner::drop -- this was originally implemented because + // we could had already scheduled the deletion at the time. + // + // FIXME: this is not true anymore, we can safely evict wanted deleted files. } else if can_evict && evict { // If timeline is alive, we can construct a span with IDs for this function. let span = self.timeline.upgrade().map(|timeline| { @@ -1051,7 +1054,7 @@ impl LayerInner { crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { let _g = span.map(|s| s.entered()); - // if LayerInner is already dropped here, do nothing because the garbage collection + // if LayerInner is already dropped here, do nothing because the delete on drop // has already ran while we were in queue let Some(this) = this.upgrade() else { LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone); @@ -1455,35 +1458,37 @@ impl From for Layer { } } -use metrics::{IntCounter, IntCounterVec}; +use metrics::IntCounter; -struct LayerImplMetrics { +pub(crate) struct LayerImplMetrics { started_evictions: IntCounter, completed_evictions: IntCounter, - cancelled_evictions: IntCounterVec, + cancelled_evictions: enum_map::EnumMap, - started_gcs: IntCounter, - completed_gcs: IntCounter, - failed_gcs: IntCounterVec, + started_deletes: IntCounter, + completed_deletes: IntCounter, + failed_deletes: enum_map::EnumMap, - rare_counters: IntCounterVec, + rare_counters: enum_map::EnumMap, + inits_cancelled: metrics::core::GenericCounter, } impl Default for LayerImplMetrics { fn default() -> Self { - let evictions = metrics::register_int_counter_vec!( - "pageserver_layer_evictions_count", - "Evictions started and completed in the Layer implementation", - &["state"] + use enum_map::Enum; + + // reminder: these will be pageserver_layer_* with "_total" suffix + + let started_evictions = metrics::register_int_counter!( + "pageserver_layer_started_evictions", + "Evictions started in the Layer implementation" + ) + .unwrap(); + let completed_evictions = metrics::register_int_counter!( + "pageserver_layer_completed_evictions", + "Evictions completed in the Layer implementation" ) .unwrap(); - - let started_evictions = evictions - .get_metric_with_label_values(&["started"]) - .unwrap(); - let completed_evictions = evictions - .get_metric_with_label_values(&["completed"]) - .unwrap(); let cancelled_evictions = metrics::register_int_counter_vec!( "pageserver_layer_cancelled_evictions_count", @@ -1492,24 +1497,36 @@ impl Default for LayerImplMetrics { ) .unwrap(); - // reminder: this will be pageserver_layer_gcs_count_total with "_total" suffix - let gcs = metrics::register_int_counter_vec!( - "pageserver_layer_gcs_count", - "Garbage collections started and completed in the Layer implementation", - &["state"] + let cancelled_evictions = enum_map::EnumMap::from_array(std::array::from_fn(|i| { + let reason = EvictionCancelled::from_usize(i); + let s = reason.as_str(); + cancelled_evictions.with_label_values(&[s]) + })); + + let started_deletes = metrics::register_int_counter!( + "pageserver_layer_started_deletes", + "Deletions on drop pending in the Layer implementation" + ) + .unwrap(); + let completed_deletes = metrics::register_int_counter!( + "pageserver_layer_completed_deletes", + "Deletions on drop completed in the Layer implementation" ) .unwrap(); - let started_gcs = gcs.get_metric_with_label_values(&["pending"]).unwrap(); - let completed_gcs = gcs.get_metric_with_label_values(&["completed"]).unwrap(); - - let failed_gcs = metrics::register_int_counter_vec!( - "pageserver_layer_failed_gcs_count", - "Different reasons for garbage collections to have failed", + let failed_deletes = metrics::register_int_counter_vec!( + "pageserver_layer_failed_deletes_count", + "Different reasons for deletions on drop to have failed", &["reason"] ) .unwrap(); + let failed_deletes = enum_map::EnumMap::from_array(std::array::from_fn(|i| { + let reason = DeleteFailed::from_usize(i); + let s = reason.as_str(); + failed_deletes.with_label_values(&[s]) + })); + let rare_counters = metrics::register_int_counter_vec!( "pageserver_layer_assumed_rare_count", "Times unexpected or assumed rare event happened", @@ -1517,16 +1534,29 @@ impl Default for LayerImplMetrics { ) .unwrap(); + let rare_counters = enum_map::EnumMap::from_array(std::array::from_fn(|i| { + let event = RareEvent::from_usize(i); + let s = event.as_str(); + rare_counters.with_label_values(&[s]) + })); + + let inits_cancelled = metrics::register_int_counter!( + "pageserver_layer_inits_cancelled_count", + "Times Layer initialization was cancelled", + ) + .unwrap(); + Self { started_evictions, completed_evictions, cancelled_evictions, - started_gcs, - completed_gcs, - failed_gcs, + started_deletes, + completed_deletes, + failed_deletes, rare_counters, + inits_cancelled, } } } @@ -1539,57 +1569,33 @@ impl LayerImplMetrics { self.completed_evictions.inc(); } fn inc_eviction_cancelled(&self, reason: EvictionCancelled) { - self.cancelled_evictions - .get_metric_with_label_values(&[reason.as_str()]) - .unwrap() - .inc() + self.cancelled_evictions[reason].inc() } - fn inc_started_gcs(&self) { - self.started_gcs.inc(); + fn inc_started_deletes(&self) { + self.started_deletes.inc(); } - fn inc_completed_gcs(&self) { - self.completed_gcs.inc(); + fn inc_completed_deletes(&self) { + self.completed_deletes.inc(); } - fn inc_gcs_failed(&self, reason: GcFailed) { - self.failed_gcs - .get_metric_with_label_values(&[reason.as_str()]) - .unwrap() - .inc(); + fn inc_deletes_failed(&self, reason: DeleteFailed) { + self.failed_deletes[reason].inc(); } - /// Counted separatedly from failed gcs because we will complete the gc attempt regardless of - /// failure to delete local file. - fn inc_gc_removes_failed(&self) { - self.rare_counters - .get_metric_with_label_values(&["gc_remove_failed"]) - .unwrap() - .inc(); + /// Counted separatedly from failed layer deletes because we will complete the layer deletion + /// attempt regardless of failure to delete local file. + fn inc_delete_removes_failed(&self) { + self.rare_counters[RareEvent::RemoveOnDropFailed].inc(); } - /// Expected rare because requires a race with `evict_blocking` and - /// `get_or_maybe_download`. + /// Expected rare because requires a race with `evict_blocking` and `get_or_maybe_download`. fn inc_retried_get_or_maybe_download(&self) { - self.rare_counters - .get_metric_with_label_values(&["retried_gomd"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::RetriedGetOrMaybeDownload].inc(); } - /// Expected rare because cancellations are unexpected - fn inc_download_completed_without_requester(&self) { - self.rare_counters - .get_metric_with_label_values(&["download_completed_without"]) - .unwrap() - .inc(); - } - - /// Expected rare because cancellations are unexpected + /// Expected rare because cancellations are unexpected, and failures are unexpected fn inc_download_failed_without_requester(&self) { - self.rare_counters - .get_metric_with_label_values(&["download_failed_without"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::DownloadFailedWithoutRequester].inc(); } /// The Weak in ResidentOrWantedEvicted::WantedEvicted was successfully upgraded. @@ -1597,37 +1603,30 @@ impl LayerImplMetrics { /// If this counter is always zero, we should replace ResidentOrWantedEvicted type with an /// Option. fn inc_raced_wanted_evicted_accesses(&self) { - self.rare_counters - .get_metric_with_label_values(&["raced_wanted_evicted"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::UpgradedWantedEvicted].inc(); } - /// These are only expected for [`Self::inc_download_completed_without_requester`] amount when + /// These are only expected for [`Self::inc_init_cancelled`] amount when /// running with remote storage. fn inc_init_needed_no_download(&self) { - self.rare_counters - .get_metric_with_label_values(&["init_needed_no_download"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::InitWithoutDownload].inc(); } /// Expected rare because all layer files should be readable and good fn inc_permanent_loading_failures(&self) { - self.rare_counters - .get_metric_with_label_values(&["permanent_loading_failure"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::PermanentLoadingFailure].inc(); } fn inc_broadcast_lagged(&self) { - self.rare_counters - .get_metric_with_label_values(&["broadcast_lagged"]) - .unwrap() - .inc(); + self.rare_counters[RareEvent::EvictAndWaitLagged].inc(); + } + + fn inc_init_cancelled(&self) { + self.inits_cancelled.inc() } } +#[derive(enum_map::Enum)] enum EvictionCancelled { LayerGone, TimelineGone, @@ -1656,19 +1655,47 @@ impl EvictionCancelled { } } -enum GcFailed { +#[derive(enum_map::Enum)] +enum DeleteFailed { TimelineGone, DeleteSchedulingFailed, } -impl GcFailed { +impl DeleteFailed { fn as_str(&self) -> &'static str { match self { - GcFailed::TimelineGone => "timeline_gone", - GcFailed::DeleteSchedulingFailed => "delete_scheduling_failed", + DeleteFailed::TimelineGone => "timeline_gone", + DeleteFailed::DeleteSchedulingFailed => "delete_scheduling_failed", } } } -static LAYER_IMPL_METRICS: once_cell::sync::Lazy = +#[derive(enum_map::Enum)] +enum RareEvent { + RemoveOnDropFailed, + RetriedGetOrMaybeDownload, + DownloadFailedWithoutRequester, + UpgradedWantedEvicted, + InitWithoutDownload, + PermanentLoadingFailure, + EvictAndWaitLagged, +} + +impl RareEvent { + fn as_str(&self) -> &'static str { + use RareEvent::*; + + match self { + RemoveOnDropFailed => "remove_on_drop_failed", + RetriedGetOrMaybeDownload => "retried_gomd", + DownloadFailedWithoutRequester => "download_failed_without", + UpgradedWantedEvicted => "raced_wanted_evicted", + InitWithoutDownload => "init_needed_no_download", + PermanentLoadingFailure => "permanent_loading_failure", + EvictAndWaitLagged => "broadcast_lagged", + } + } +} + +pub(crate) static LAYER_IMPL_METRICS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(LayerImplMetrics::default); diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 729b8cedfb..f472bae7e6 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -482,7 +482,7 @@ impl Timeline { .map(|ancestor| ancestor.timeline_id) } - /// Lock and get timeline's GC cuttof + /// Lock and get timeline's GC cutoff pub fn get_latest_gc_cutoff_lsn(&self) -> RcuReadGuard { self.latest_gc_cutoff_lsn.read() } @@ -3984,7 +3984,7 @@ impl Timeline { // for details. This will block until the old value is no longer in use. // // The GC cutoff should only ever move forwards. - { + let waitlist = { let write_guard = self.latest_gc_cutoff_lsn.lock_for_write(); ensure!( *write_guard <= new_gc_cutoff, @@ -3992,8 +3992,9 @@ impl Timeline { *write_guard, new_gc_cutoff ); - write_guard.store_and_unlock(new_gc_cutoff).wait(); - } + write_guard.store_and_unlock(new_gc_cutoff) + }; + waitlist.wait().await; info!("GC starting"); diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 10588ee884..913fa36d28 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -248,7 +248,7 @@ impl LayerManager { // map index without actually rebuilding the index. updates.remove_historic(desc); mapping.remove(layer); - layer.garbage_collect_on_drop(); + layer.delete_on_drop(); } pub(crate) fn contains(&self, layer: &Layer) -> bool { diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index b58b883ab6..7a6443361e 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -610,9 +610,11 @@ impl Drop for VirtualFile { slot.recently_used.store(false, Ordering::Relaxed); // there is also operation "close-by-replace" for closes done on eviction for // comparison. - STORAGE_IO_TIME_METRIC - .get(StorageIoOperation::Close) - .observe_closure_duration(|| drop(slot_guard.file.take())); + if let Some(fd) = slot_guard.file.take() { + STORAGE_IO_TIME_METRIC + .get(StorageIoOperation::Close) + .observe_closure_duration(|| drop(fd)); + } } } } diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 9c2bb2db11..67daf5f901 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -840,7 +840,7 @@ def test_compaction_waits_for_upload( ), "there should be one L1 after L0 => L1 compaction (without #5863 being fixed)" def layer_deletes_completed(): - m = client.get_metric_value("pageserver_layer_gcs_count_total", {"state": "completed"}) + m = client.get_metric_value("pageserver_layer_completed_deletes_total") if m is None: return 0 return int(m)